dohertj2/lmxopcua
1
0
Fork 0
Code Issues 43 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(runtime): dispose drivers on final stop, bound PostStop shutdown (A1)

Browse Source 
DriverInstanceActor.PostStop now bounds IDriver.ShutdownAsync by a
shutdownTimeout (default 5 s, logged on overrun) and then disposes the
driver (IAsyncDisposable preferred, else IDisposable) — but only on a
real stop. PreRestart flags the restart so the PostStop it triggers
skips the dispose, because a supervision restart re-uses the same
driver object (Props closure) and the host re-initialises it.

DiscoveryDriverBrowser disposes its throwaway capture/probe driver
after the bounded shutdown, including after a failed or timed-out one.

Review refs: 02/L1, 02/P4, 06/G-1 (host context), 07/L2.

Refs #731

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 06:23:39 -04:00
parent ad2b5c4044
commit eb23053700
4 changed files with 428 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Core/ZB.MOM.WW.OtOpcUa.Commons/Browsing/DiscoveryDriverBrowser.cs
+26 -1
View File
@@ -188,6 +188,13 @@ public sealed class DiscoveryDriverBrowser : IUniversalDriverBrowser
}
}
/// <summary>
/// Final teardown of a throwaway capture/probe driver: a bounded <see cref="IDriver.ShutdownAsync"/>
/// followed by a bounded dispose. The dispose is NOT optional and runs even after a failed or timed-out
/// shutdown: a Galaxy driver's <c>ShutdownAsync</c> releases none of its gRPC runtime (channels, MX
/// session, stream loops) — only <c>DisposeAsync</c> does — so a browse that stopped at shutdown leaked a
/// full runtime per <c>/raw</c> browse and per <see cref="CanBrowse"/> probe (02/L1, 06/G-1).
/// </summary>
private async Task BoundedShutdownAsync(IDriver driver, string driverType)
{
try
@@ -199,7 +206,25 @@ public sealed class DiscoveryDriverBrowser : IUniversalDriverBrowser
catch (Exception ex)
{
_logger.LogWarning(ex,
"Browse-capture shutdown for {DriverType} failed/timed out; instance abandoned", driverType);
"Browse-capture shutdown for {DriverType} failed/timed out; disposing the instance anyway", driverType);
}
try
{
switch (driver)
{
case IAsyncDisposable asyncDisposable:
await asyncDisposable.DisposeAsync().AsTask().WaitAsync(_shutdownTimeout).ConfigureAwait(false);
break;
case IDisposable disposable:
disposable.Dispose();
break;
}
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Browse-capture dispose for {DriverType} failed/timed out; instance abandoned", driverType);
}
}
src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverInstanceActor.cs
+104 -5
View File
@@ -184,6 +184,16 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
/// </remarks>
public static readonly TimeSpan DefaultCommsLossTimeout = TimeSpan.FromSeconds(90);
/// <summary>
/// Bound on the driver's <see cref="IDriver.ShutdownAsync"/> (and its dispose) when the child stops or
/// restarts, and on the teardown that precedes every reconnect attempt. <c>PostStop</c> runs on a
/// dispatcher thread and blocks on it, so a driver whose shutdown waits on a dead socket / COM proxy used
/// to pin that thread for as long as the wait took — a few such stops starved the default dispatcher and
/// wedged <c>CoordinatedShutdown</c> and the host's own restart cascade. A shutdown that overruns is
/// abandoned (logged), never awaited further.
/// </summary>
public static readonly TimeSpan DefaultShutdownTimeout = TimeSpan.FromSeconds(5);
private readonly IDriver _driver;
/// <summary>Phase 6.1 resilience seam: every guarded driver-capability call this actor makes is
@@ -204,6 +214,18 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
/// <summary>Comms-loss watchdog threshold; <see cref="TimeSpan.Zero"/> or negative disables it.</summary>
private readonly TimeSpan _commsLossTimeout;
/// <summary>Bound on the driver shutdown / dispose; see <see cref="DefaultShutdownTimeout"/>.</summary>
private readonly TimeSpan _shutdownTimeout;
/// <summary>
/// Set by <see cref="PreRestart"/> so the <see cref="PostStop"/> it triggers skips the dispose. A
/// supervision restart recreates the actor object around the SAME <see cref="IDriver"/> (the Props closure
/// captured it) and the host then re-initialises that driver — disposing it here would hand the new
/// incarnation a dead object. Only a real stop (deploy removal, operator Restart, respawn on config change,
/// host shutdown) disposes.
/// </summary>
private bool _restarting;
/// <summary>When the actor last entered <c>Connected</c>. The comms-loss watchdog's baseline for a
/// driver that has never reported a successful read, so "connected but never read anything" is
/// measurable rather than indefinitely undefined.</summary>
@@ -307,6 +329,9 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
/// <param name="commsLossTimeout">Optional comms-loss watchdog threshold; defaults to
/// <see cref="DefaultCommsLossTimeout"/>. <see cref="TimeSpan.Zero"/> disables the watchdog — used by
/// tests that need a Connected driver to stay Connected while reporting non-Healthy.</param>
/// <param name="shutdownTimeout">Optional bound on the driver's shutdown / dispose on stop and on the
/// pre-reconnect teardown; defaults to <see cref="DefaultShutdownTimeout"/>. Exists so a test can prove a
/// hung shutdown is abandoned without waiting 5 s.</param>
/// <returns>A <see cref="Akka.Actor.Props"/> instance configured to create the wrapped <see cref="DriverInstanceActor"/>.</returns>
public static Props Props(
IDriver driver,
@@ -316,7 +341,8 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
string? clusterId = null,
IDriverCapabilityInvoker? invoker = null,
TimeSpan? healthPollInterval = null,
TimeSpan? commsLossTimeout = null) =>
TimeSpan? commsLossTimeout = null,
TimeSpan? shutdownTimeout = null) =>
Akka.Actor.Props.Create(() => new DriverInstanceActor(
driver,
reconnectInterval ?? DefaultReconnectInterval,
@@ -325,7 +351,8 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
clusterId ?? string.Empty,
invoker,
healthPollInterval,
commsLossTimeout));
commsLossTimeout,
shutdownTimeout));
/// <summary>
/// Returns true when the driver should boot in DEV-STUB mode based on host platform and
@@ -362,6 +389,8 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
/// subscription reconcile; defaults to <see cref="HealthPollInterval"/> when null.</param>
/// <param name="commsLossTimeout">Comms-loss watchdog threshold; defaults to
/// <see cref="DefaultCommsLossTimeout"/> when null, and is disabled when zero or negative.</param>
/// <param name="shutdownTimeout">Bound on the driver's shutdown / dispose; defaults to
/// <see cref="DefaultShutdownTimeout"/> when null.</param>
public DriverInstanceActor(
IDriver driver,
TimeSpan reconnectInterval,
@@ -370,7 +399,8 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
string? clusterId = null,
IDriverCapabilityInvoker? invoker = null,
TimeSpan? healthPollInterval = null,
TimeSpan? commsLossTimeout = null)
TimeSpan? commsLossTimeout = null,
TimeSpan? shutdownTimeout = null)
{
_driver = driver;
_invoker = invoker ?? NullDriverCapabilityInvoker.Instance;
@@ -380,6 +410,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
_reconnectInterval = reconnectInterval;
_healthPollInterval = healthPollInterval ?? HealthPollInterval;
_commsLossTimeout = commsLossTimeout ?? DefaultCommsLossTimeout;
_shutdownTimeout = shutdownTimeout is { } bound && bound > TimeSpan.Zero ? bound : DefaultShutdownTimeout;
OtOpcUaTelemetry.DriverInstanceLifecycle.Add(1,
new KeyValuePair<string, object?>("event", startStubbed ? "spawn_stub" : "spawn"),
new KeyValuePair<string, object?>("driver_type", driver.DriverType));
@@ -421,6 +452,18 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
Context.Parent.Tell(new ReprovisionRequested(_driverInstanceId, reason.GetType().Name));
}
/// <summary>
/// Flags the restart before the base implementation runs <see cref="PostStop"/>, so that PostStop
/// shuts the driver down (the host re-initialises it on <see cref="ReprovisionRequested"/>) but does
/// NOT dispose it — see <see cref="_restarting"/>.
/// </summary>
/// <inheritdoc />
protected override void PreRestart(Exception reason, object message)
{
_restarting = true;
base.PreRestart(reason, message);
}
private void Stubbed()
{
// Stubbed drivers accept the standard message contracts but return deterministic
@@ -1547,10 +1590,66 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
DetachRediscoverySource();
DetachHostStatusSource();
DetachConnectionLossSource();
try { _driver.ShutdownAsync(CancellationToken.None).GetAwaiter().GetResult(); }
catch (Exception ex) { _log.Warning(ex, "DriverInstance {Id}: ShutdownAsync threw on PostStop", _driverInstanceId); }
// BOUNDED. PostStop runs on a dispatcher thread and this blocks it, so an unbounded wait on a driver
// whose shutdown hangs against a dead socket / COM proxy pinned that thread for the whole wait, and a
// few such stops starved the dispatcher. A shutdown that overruns the bound is abandoned.
using (var cts = new CancellationTokenSource(_shutdownTimeout))
{
try { _driver.ShutdownAsync(cts.Token).WaitAsync(_shutdownTimeout).GetAwaiter().GetResult(); }
catch (TimeoutException)
{
_log.Warning("DriverInstance {Id}: ShutdownAsync did not complete within {Bound} on PostStop; abandoning it",
_driverInstanceId, _shutdownTimeout);
}
catch (Exception ex) { _log.Warning(ex, "DriverInstance {Id}: ShutdownAsync threw on PostStop", _driverInstanceId); }
}
// Dispose ONLY on a real stop. A supervision restart (PreRestart → this PostStop) re-uses the same driver
// object for the next incarnation, which the host re-initialises. Everything else that stops a child —
// deploy removal, operator Restart, respawn on a config change, host shutdown — is final, and the driver
// is never touched again: before this, nothing in the runtime ever disposed an IDriver, so a Calculation
// driver's compiled-script AssemblyLoadContexts and an OpcUaClient driver's LoggerFactory / gate / CTS
// leaked once per respawn (02/L1). Bounded like the shutdown, and runs even after a timed-out shutdown —
// leaking the object is strictly worse than disposing one whose shutdown is still wedged.
if (!_restarting) DisposeDriver();
OtOpcUaTelemetry.DriverInstanceLifecycle.Add(1,
new KeyValuePair<string, object?>("event", "stop"),
new KeyValuePair<string, object?>("driver_type", _driver.DriverType));
}
/// <summary>Final teardown of the wrapped driver: <see cref="IAsyncDisposable"/> preferred, else
/// <see cref="IDisposable"/>, never both. A driver implementing neither needs nothing here.</summary>
private void DisposeDriver()
{
try
{
switch (_driver)
{
case IAsyncDisposable asyncDisposable:
asyncDisposable.DisposeAsync().AsTask().WaitAsync(_shutdownTimeout).GetAwaiter().GetResult();
break;
case IDisposable disposable:
disposable.Dispose();
break;
default:
return;
}
_log.Info("DriverInstance {Id}: disposed {Type} driver on stop", _driverInstanceId, _driver.DriverType);
OtOpcUaTelemetry.DriverInstanceLifecycle.Add(1,
new KeyValuePair<string, object?>("event", "dispose"),
new KeyValuePair<string, object?>("driver_type", _driver.DriverType));
}
catch (TimeoutException)
{
_log.Warning("DriverInstance {Id}: DisposeAsync did not complete within {Bound} on PostStop; abandoning it",
_driverInstanceId, _shutdownTimeout);
}
catch (Exception ex)
{
_log.Warning(ex, "DriverInstance {Id}: disposing the driver threw on PostStop", _driverInstanceId);
}
}
}