fix(runtime): re-provision a driver child after a supervision restart (archreview 03/S6, S16)

This commit is contained in:
Joseph Doherty
2026-08-20 09:56:05 -04:00
parent d97e441979
commit 2b9a397b2d
4 changed files with 81 additions and 1 deletions
@@ -595,3 +595,13 @@ discipline for this assembly).
works: with T3 reverted this test fails on the *positive control*, not on the absence.
**Not covered, and worth a reviewer's eye:** the `_desiredByDriver` union's `_children.Keys` arm
(a running child absent from BOTH ref maps) is not reachable through this seeder and is untested.
### T6 — supervision reprovision hook
`DriverInstanceActor.PostRestart` (calling `base.PostRestart(reason)` first, so `PreStart` re-attaches
the rediscovery / host-status sources and restarts the health timer) → `ReprovisionRequested` →
`DriverHostActor.HandleReprovisionRequested`, registered in **all three** `Receive` blocks that already
accept `RestartDriver`. The falsification is T7, below.
Gate: `--filter "…~DriverHostActor|…~DriverInstanceActor"` → **183/183 passed** (179 before this task
plus T5's four), no test moved.
@@ -40,7 +40,7 @@
{
"id": "T6",
"subject": "Supervision reprovision hook: DriverInstanceActor.PostRestart -> ReprovisionRequested -> host re-init + re-subscribe (03/S6, S16)",
"status": "pending",
"status": "done",
"blockedBy": [
"T3"
]
@@ -1088,6 +1088,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied);
Receive<DriverInstanceActor.ApplyResult>(HandleApplyResult);
Receive<RestartDriver>(HandleRestartDriver);
Receive<DriverInstanceActor.ReprovisionRequested>(HandleReprovisionRequested);
Receive<ReconnectDriver>(HandleReconnectDriver);
Receive<RouteNodeWrite>(HandleRouteNodeWrite);
Receive<RouteNativeAlarmAck>(HandleRouteNativeAlarmAck);
@@ -1143,6 +1144,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied);
Receive<DriverInstanceActor.ApplyResult>(HandleApplyResult);
Receive<RestartDriver>(HandleRestartDriver);
Receive<DriverInstanceActor.ReprovisionRequested>(HandleReprovisionRequested);
Receive<ReconnectDriver>(HandleReconnectDriver);
Receive<RouteNodeWrite>(HandleRouteNodeWrite);
Receive<RouteNativeAlarmAck>(HandleRouteNativeAlarmAck);
@@ -1725,6 +1727,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<GetDiagnostics>(HandleGetDiagnostics);
Receive<RetryConfigDbConnection>(_ => TryRecoverFromStale());
Receive<RestartDriver>(HandleRestartDriver);
Receive<DriverInstanceActor.ReprovisionRequested>(HandleReprovisionRequested);
Receive<ReconnectDriver>(HandleReconnectDriver);
Receive<RedundancyStateChanged>(OnRedundancyStateChanged);
// The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes).
@@ -2991,6 +2994,40 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
SpawnChild(entry.Spec);
}
/// <summary>
/// Re-provision a driver child that Akka's supervision just RESTARTED in place (archreview 03/S6,
/// 03/S16) — the third and least visible of the three incarnation routes.
/// </summary>
/// <remarks>
/// A restart recreates the actor object around the SAME driver instance (the Props closure captured
/// it) without going through <see cref="SpawnChild"/>, so neither the <c>InitializeRequested</c> nor
/// the desired-subscription send that a spawn performs ever happens — and the child's
/// <see cref="IActorRef"/> is unchanged, so <c>_children</c> needs no maintenance and the host has no
/// other notification that a new incarnation exists. Its <c>PreRestart</c> ran <c>PostStop</c>, which
/// called <c>ShutdownAsync</c> on the driver, so re-initialising is not optional: without it the
/// child sits in <c>Connecting</c> forever around a shut-down driver.
/// <para>
/// The dependency-consumer mux adapter is a SEPARATE child of this host and is unaffected by the
/// instance actor restarting, so it is deliberately not touched here.
/// </para>
/// </remarks>
/// <param name="msg">The restarted child's self-announcement.</param>
private void HandleReprovisionRequested(DriverInstanceActor.ReprovisionRequested msg)
{
// Already stopped (a restart racing a reconcile that removed the driver) — nothing to provision.
if (!_children.TryGetValue(msg.DriverInstanceId, out var entry)) return;
// A crash that Akka silently restarted is otherwise invisible to operators; name it.
_log.Warning(
"DriverHost {Node}: driver child {Id} was restarted by supervision after {Reason} — re-provisioning",
_localNode, msg.DriverInstanceId, msg.Reason);
// A stub child has no real driver to initialise (its Stubbed behaviour no-ops InitializeRequested
// anyway), but it still stores a desired set, so only the initialise is conditional.
if (!entry.Stubbed) entry.Actor.Tell(new DriverInstanceActor.InitializeRequested(entry.Spec.DriverConfig));
SendDesiredSubscriptions(entry.Actor, msg.DriverInstanceId);
}
private void HandleReconnectDriver(ReconnectDriver msg)
{
// DPS broadcast — only act if this node hosts the requested instance.
@@ -42,6 +42,25 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
/// restored → Good) — independently of alarm transitions, since a comms-lost driver emits no alarm
/// events. Fire-and-forget, mirroring <see cref="DeltaApplied"/>.</summary>
public sealed record ConnectivityChanged(string DriverInstanceId, bool Connected);
/// <summary>
/// Sent to the parent from <see cref="PostRestart"/> so the host re-provisions THIS incarnation
/// (archreview 03/S6, 03/S16).
/// </summary>
/// <remarks>
/// A supervision restart recreates the actor object in place: the desired subscription sets are gone
/// (they are instance fields), and <c>PreRestart</c> has already run <see cref="PostStop"/> — which
/// called <see cref="IDriver.ShutdownAsync"/> on the driver object the Props closure re-uses. Nothing
/// else re-sends <see cref="InitializeRequested"/>: its only sender is
/// <c>DriverHostActor.SpawnChild</c>, and a restart does NOT respawn (the child's
/// <see cref="IActorRef"/> is unchanged, so the host is not otherwise notified that a new incarnation
/// exists at all). Without this the restarted child sits in <c>Connecting</c> forever around a
/// shut-down driver, with nothing subscribed — silently, since Akka logs the fault but the recovery
/// is invisible.
/// </remarks>
/// <param name="DriverInstanceId">The driver instance that was restarted.</param>
/// <param name="Reason">The type name of the exception that tripped supervision, for the host's log.</param>
public sealed record ReprovisionRequested(string DriverInstanceId, string Reason);
public sealed record ApplyDelta(string DriverConfigJson, CorrelationId Correlation);
public sealed record ApplyResult(bool Success, string? Reason, CorrelationId Correlation);
/// <summary>
@@ -375,6 +394,20 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
Timers.StartPeriodicTimer("health-poll", HealthPollTick.Instance, _healthPollInterval);
}
/// <inheritdoc />
/// <remarks>
/// Ask the host to re-provision this incarnation — see <see cref="ReprovisionRequested"/> for why a
/// supervision restart otherwise leaves a dead child. <c>base.PostRestart</c> runs first because it
/// runs <see cref="PreStart"/>, which re-attaches the rediscovery / host-status sources and restarts
/// the health timer; the reprovision is the part <see cref="PreStart"/> cannot do, since only the
/// host knows this driver's config and desired set.
/// </remarks>
protected override void PostRestart(Exception reason)
{
base.PostRestart(reason);
Context.Parent.Tell(new ReprovisionRequested(_driverInstanceId, reason.GetType().Name));
}
private void Stubbed()
{
// Stubbed drivers accept the standard message contracts but return deterministic