dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(runtime): rate-limit the supervision reprovision — deferred, not dropped (review F3)

Browse Source 
The default decider restarts a faulting child immediately and without limit, and
the hook makes every restart do ShutdownAsync+InitializeAsync+SubscribeAsync, so a
driver faulting on every value became a reconnect storm. Per-driver last-reprovision
stamp + DefaultReprovisionBackoff (10s, matched to DefaultReconnectInterval);
a held request is re-delivered by a single per-driver timer carrying the original
incarnation ref, so the F2 identity check still applies.

Seen RED with the backoff disabled: InitializeCount == 3 (second applied
immediately) where the test requires 2-then-3.

Also: the _children.Keys union-arm test (seeder gained taglessDrivers) and the
_desiredByDriver xmldoc note that it lingers on last-known-good after a failed apply.
This commit is contained in:
Joseph Doherty
2026-08-20 10:39:13 -04:00
parent 2b7100cf19
commit 0793b0cba8
4 changed files with 212 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs
+67 -7
View File
@@ -82,6 +82,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// </remarks>
public static readonly TimeSpan DefaultApplyWedgeTimeout = TimeSpan.FromMinutes(5);
/// <summary>Minimum interval between two supervision reprovisions of the SAME driver child (archreview
/// review F3). Akka's default decider restarts a faulting child immediately and without limit, and the
/// reprovision hook makes every restart do ShutdownAsync + InitializeAsync + SubscribeAsync — so an
/// un-limited hook turns a driver that faults on every value into a reconnect storm against its device.
/// Matched to <see cref="DriverInstanceActor.DefaultReconnectInterval"/>: the reprovision drives the same
/// connect work, so it should not run more often than an ordinary reconnect attempt.</summary>
public static readonly TimeSpan DefaultReprovisionBackoff = TimeSpan.FromSeconds(10);
/// <summary>
/// Cache key used when an artifact carries no cluster scoping (single-cluster or unscoped
/// deployments). A literal rather than an empty string so the row is visibly deliberate when
@@ -357,7 +365,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// push, <see cref="SpawnChild"/> and the supervision-reprovision handler all send from here, so a
/// child obtained by ANY route carries the same set. Cleared and repopulated whole on every apply,
/// alongside the routing maps, so renames/removals can never leave a stale entry. Empty until the
/// first apply — a child spawned before any apply is handed nothing, which is what it should have.</summary>
/// first apply — a child spawned before any apply is handed nothing, which is what it should have.
/// <para>It is only ever rebuilt by a SUCCESSFUL apply, so after a failed one it deliberately lingers on
/// the last-good composition — matching the address space, drivers and subscriptions that also keep
/// serving through a failed apply (#485/#486). A respawn or reprovision in that window therefore hands
/// the child the last-known-good set, not an empty one.</para></summary>
private readonly Dictionary<string, DesiredSubscriptionSet> _desiredByDriver = new(StringComparer.Ordinal);
/// <summary>
@@ -413,6 +425,15 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// <see cref="DefaultApplyWedgeTimeout"/>).</summary>
private readonly TimeSpan _applyWedgeTimeout;
/// <summary>Per-driver rate limit for the supervision-reprovision hook (see
/// <see cref="DefaultReprovisionBackoff"/>).</summary>
private readonly TimeSpan _reprovisionBackoff;
/// <summary>When each driver was last reprovisioned after a supervision restart, so a crash loop is
/// paced rather than amplified. Bounded by the number of driver children; an entry for a driver that is
/// later removed simply stops being consulted (the handler resolves <c>_children</c> first).</summary>
private readonly Dictionary<string, DateTime> _lastReprovisionUtc = new(StringComparer.Ordinal);
/// <summary>
/// Host of this node's LocalDb replication partner — the only node that holds a copy of this
/// node's alarm queue, and therefore the only node it may ever stand down in favour of.
@@ -554,7 +575,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
IDriverPrimaryRoleSource? primaryRoleSource = null,
IHistorizationRoleView? historizationRoleView = null,
Func<CommonsNodeId, bool>? sharesHistorizationScopeWithLocal = null,
TimeSpan? applyWedgeTimeout = null) =>
TimeSpan? applyWedgeTimeout = null,
TimeSpan? reprovisionBackoff = null) =>
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
@@ -566,7 +588,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider,
deploymentArtifactCache, redundancyRoleView, replicationPeerHost, fetchAndCacheMode, artifactFetcher,
alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
sharesHistorizationScopeWithLocal, applyWedgeTimeout));
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff));
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
/// <param name="dbFactory">Database context factory for configuration database access.</param>
@@ -652,9 +674,11 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
IDriverPrimaryRoleSource? primaryRoleSource = null,
IHistorizationRoleView? historizationRoleView = null,
Func<CommonsNodeId, bool>? sharesHistorizationScopeWithLocal = null,
TimeSpan? applyWedgeTimeout = null)
TimeSpan? applyWedgeTimeout = null,
TimeSpan? reprovisionBackoff = null)
{
_applyWedgeTimeout = applyWedgeTimeout ?? DefaultApplyWedgeTimeout;
_reprovisionBackoff = reprovisionBackoff ?? DefaultReprovisionBackoff;
_historizationRoleView = historizationRoleView;
_sharesHistorizationScopeWithLocal = sharesHistorizationScopeWithLocal;
_telemetryHub = telemetryHub;
@@ -1089,6 +1113,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<DriverInstanceActor.ApplyResult>(HandleApplyResult);
Receive<RestartDriver>(HandleRestartDriver);
Receive<DriverInstanceActor.ReprovisionRequested>(HandleReprovisionRequested);
Receive<ReprovisionDeferred>(m => ReprovisionChild(m.Request, m.Child));
Receive<ReconnectDriver>(HandleReconnectDriver);
Receive<RouteNodeWrite>(HandleRouteNodeWrite);
Receive<RouteNativeAlarmAck>(HandleRouteNativeAlarmAck);
@@ -1145,6 +1170,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<DriverInstanceActor.ApplyResult>(HandleApplyResult);
Receive<RestartDriver>(HandleRestartDriver);
Receive<DriverInstanceActor.ReprovisionRequested>(HandleReprovisionRequested);
Receive<ReprovisionDeferred>(m => ReprovisionChild(m.Request, m.Child));
Receive<ReconnectDriver>(HandleReconnectDriver);
Receive<RouteNodeWrite>(HandleRouteNodeWrite);
Receive<RouteNativeAlarmAck>(HandleRouteNativeAlarmAck);
@@ -1728,6 +1754,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<RetryConfigDbConnection>(_ => TryRecoverFromStale());
Receive<RestartDriver>(HandleRestartDriver);
Receive<DriverInstanceActor.ReprovisionRequested>(HandleReprovisionRequested);
Receive<ReprovisionDeferred>(m => ReprovisionChild(m.Request, m.Child));
Receive<ReconnectDriver>(HandleReconnectDriver);
Receive<RedundancyStateChanged>(OnRedundancyStateChanged);
// The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes).
@@ -3012,7 +3039,20 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// </para>
/// </remarks>
/// <param name="msg">The restarted child's self-announcement.</param>
private void HandleReprovisionRequested(DriverInstanceActor.ReprovisionRequested msg)
private void HandleReprovisionRequested(DriverInstanceActor.ReprovisionRequested msg) =>
ReprovisionChild(msg, Sender);
/// <summary>A reprovision held back by <see cref="_reprovisionBackoff"/>, carrying the incarnation it
/// was requested by so the identity check still applies when it is re-delivered by the timer.</summary>
/// <param name="Request">The original request.</param>
/// <param name="Child">The incarnation that asked; re-validated against <c>_children</c> on delivery,
/// so a driver restarted or removed in the meantime discards it.</param>
private sealed record ReprovisionDeferred(
DriverInstanceActor.ReprovisionRequested Request, IActorRef Child);
private static string ReprovisionTimerKey(string driverInstanceId) => "reprovision-" + driverInstanceId;
private void ReprovisionChild(DriverInstanceActor.ReprovisionRequested msg, IActorRef child)
{
// Already stopped (a restart racing a reconcile that removed the driver) — nothing to provision.
if (!_children.TryGetValue(msg.DriverInstanceId, out var entry)) return;
@@ -3023,14 +3063,34 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
// where InitializeRequested IS handled — it would start a second, concurrent InitializeAsync,
// racing the dispose-before-overwrite that #527's teardown exists to serialise
// (ModbusDriver._transport, MqttDriver._connection each leak a transport on that race).
if (!Equals(Sender, entry.Actor))
if (!Equals(child, entry.Actor))
{
_log.Debug(
"DriverHost {Node}: dropping a reprovision for {Id} from a superseded incarnation ({Stale})",
_localNode, msg.DriverInstanceId, Sender);
_localNode, msg.DriverInstanceId, child);
return;
}
// Rate-limit per driver (review F3). DEFERRED, never dropped: the last request must still be
// honoured, or a child whose final crash fell inside the window is left un-provisioned forever —
// exactly the 03/S16 failure this hook exists to close. One timer key per driver, so a burst
// collapses to a single pending reprovision rather than a queue of them.
var now = DateTime.UtcNow;
if (_lastReprovisionUtc.TryGetValue(msg.DriverInstanceId, out var last) &&
now - last < _reprovisionBackoff)
{
var delay = _reprovisionBackoff - (now - last);
_log.Warning(
"DriverHost {Node}: driver child {Id} was restarted by supervision again after {Reason} " +
"within {Backoff} — deferring the re-provision by {Delay} (crash loop?)",
_localNode, msg.DriverInstanceId, msg.Reason, _reprovisionBackoff, delay);
Timers.StartSingleTimer(
ReprovisionTimerKey(msg.DriverInstanceId), new ReprovisionDeferred(msg, child), delay);
return;
}
_lastReprovisionUtc[msg.DriverInstanceId] = now;
// A crash that Akka silently restarted is otherwise invisible to operators; name it.
_log.Warning(
"DriverHost {Node}: driver child {Id} was restarted by supervision after {Reason} — re-provisioning",