test(runtime): late-burst SourceRemoved test no longer races the 5 s PostStop budget (#828)

The test parks the removed child in PostStop (ShutdownGate) to hold the
retired-but-not-terminated window open, but PostStop abandons a shutdown after
DefaultShutdownTimeout (5 s). Under a loaded suite run more than 5 s can pass
between the stop and the host processing the stale burst; the child then
terminates, the host forgets it on Terminated, and the burst sent AS it is
forwarded and re-retained. Reproduced deterministically with a 6 s sleep
before the burst. Not a production path: a terminated actor cannot send.

DriverHostActor gains a childShutdownTimeout test seam (last parameter, passed
to DriverInstanceActor.Props). The test raises it to 5 min, asserts the child
was really stopped (ShutdownCount) before the burst, checks the premise as a
positive control (no Terminated by the time the fence has passed), and expects
Terminated once the gate is released. With an 8 s sleep before the burst the
fixed test passes.
This commit is contained in:
Joseph Doherty
2026-11-11 01:38:46 -05:00
parent 5bbe1e3c7a
commit dea5b84af0
2 changed files with 59 additions and 18 deletions
@@ -491,6 +491,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// (<see cref="DriverInstanceActor.WriteAttribute.IssuedAt"/>) means the same thing on both sides.</summary>
private readonly TimeProvider _writeClock;
/// <summary>
/// Test seam: the PostStop shutdown budget every real (non-stub) child is spawned with; null =
/// <see cref=DriverInstanceActor.DefaultShutdownTimeout/>. A test that parks a stopped child in its
/// PostStop to observe the retired-but-not-terminated window raises it, so the window cannot close under
/// load before the test has looked (lmxopcua#828).
/// </summary>
private readonly TimeSpan? _childShutdownTimeout;
/// <summary>Whether any redundancy snapshot has reached this host — the HistoryUpdate gate stays
/// closed until one has (see <see cref="PrimaryGatePolicy.ShouldServeHistoryUpdate"/>).</summary>
private bool _redundancySnapshotReceived;
@@ -738,6 +746,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// Ask is derived (<see cref="DriverWriteBounds.HostAsk"/>). Null ⇒ <see cref="DriverWriteBounds.Default"/>.</param>
/// <param name="writeClock">The clock the write bounds (and the host's write Ask) run on, shared with every
/// child. Null ⇒ <see cref="TimeProvider.System"/>.</param>
/// <param name="childShutdownTimeout">Test seam: each real child's PostStop shutdown budget; null ⇒
/// <see cref="DriverInstanceActor.DefaultShutdownTimeout"/> (lmxopcua#828).</param>
/// <returns>The Akka.NET <see cref="Akka.Actor.Props"/> used to spawn this actor.</returns>
public static Props Props(
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory,
@@ -776,7 +786,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
NativeAlarmAckPreflightView? ackPreflightView = null,
DriverWriteBounds? writeBounds = null,
TimeProvider? writeClock = null,
IScriptRunner? scriptedAlarmRunner = null) =>
IScriptRunner? scriptedAlarmRunner = null,
TimeSpan? childShutdownTimeout = null) =>
// WARNING: this forwarding list is POSITIONAL, and Props.Create compiles it into an
// expression tree. Six IActorRef? parameters and several interface-typed ones mean a
// mis-ordered argument is usually type-compatible and therefore compiles clean, then binds
@@ -790,7 +801,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
alarmStateStore, telemetryHub, primaryRoleSource, historizationRoleView,
sharesHistorizationScopeWithLocal, applyWedgeTimeout, reprovisionBackoff, commsRecoveryAuditDelay,
sharesAlarmQueueWithLocal, applyOutcomeTimeout, captureWhilePrimaryUnreachable, historyUpdateRoleView,
scripting, ackPreflightView, writeBounds, writeClock, scriptedAlarmRunner));
scripting, ackPreflightView, writeBounds, writeClock, scriptedAlarmRunner, childShutdownTimeout));
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
/// <param name="dbFactory">Database context factory for configuration database access.</param>
@@ -885,6 +896,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// Ask is derived (<see cref="DriverWriteBounds.HostAsk"/>). Null ⇒ <see cref="DriverWriteBounds.Default"/>.</param>
/// <param name="writeClock">The clock the write bounds (and the host's write Ask) run on, shared with every
/// child. Null ⇒ <see cref="TimeProvider.System"/>.</param>
/// <param name="childShutdownTimeout">Test seam: each real child's PostStop shutdown budget; null ⇒
/// <see cref="DriverInstanceActor.DefaultShutdownTimeout"/> (lmxopcua#828).</param>
public DriverHostActor(
IDbContextFactory<OtOpcUaConfigDbContext>? dbFactory,
CommonsNodeId localNode,
@@ -922,8 +935,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
NativeAlarmAckPreflightView? ackPreflightView = null,
DriverWriteBounds? writeBounds = null,
TimeProvider? writeClock = null,
IScriptRunner? scriptedAlarmRunner = null)
IScriptRunner? scriptedAlarmRunner = null,
TimeSpan? childShutdownTimeout = null)
{
_childShutdownTimeout = childShutdownTimeout;
_scriptedAlarmRunner = scriptedAlarmRunner;
_writeBounds = writeBounds ?? DriverWriteBounds.Default;
_writeClock = writeClock ?? TimeProvider.System;
@@ -3957,7 +3972,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
clusterId: clusterId,
invoker: invoker,
writeTimeout: _writeBounds.Timeout,
writeClock: _writeClock),
writeClock: _writeClock,
shutdownTimeout: _childShutdownTimeout),
actorName);
child.Tell(new DriverInstanceActor.InitializeRequested(spec.DriverConfig));
}