fix(runtime): re-provision a driver child after a supervision restart (archreview 03/S6, S16)

This commit is contained in:
Joseph Doherty
2026-08-20 09:56:05 -04:00
parent d97e441979
commit 2b9a397b2d
4 changed files with 81 additions and 1 deletions
@@ -595,3 +595,13 @@ discipline for this assembly).
works: with T3 reverted this test fails on the *positive control*, not on the absence. works: with T3 reverted this test fails on the *positive control*, not on the absence.
**Not covered, and worth a reviewer's eye:** the `_desiredByDriver` union's `_children.Keys` arm **Not covered, and worth a reviewer's eye:** the `_desiredByDriver` union's `_children.Keys` arm
(a running child absent from BOTH ref maps) is not reachable through this seeder and is untested. (a running child absent from BOTH ref maps) is not reachable through this seeder and is untested.
### T6 — supervision reprovision hook
`DriverInstanceActor.PostRestart` (calling `base.PostRestart(reason)` first, so `PreStart` re-attaches
the rediscovery / host-status sources and restarts the health timer) → `ReprovisionRequested` →
`DriverHostActor.HandleReprovisionRequested`, registered in **all three** `Receive` blocks that already
accept `RestartDriver`. The falsification is T7, below.
Gate: `--filter "…~DriverHostActor|…~DriverInstanceActor"` → **183/183 passed** (179 before this task
plus T5's four), no test moved.
@@ -40,7 +40,7 @@
{ {
"id": "T6", "id": "T6",
"subject": "Supervision reprovision hook: DriverInstanceActor.PostRestart -> ReprovisionRequested -> host re-init + re-subscribe (03/S6, S16)", "subject": "Supervision reprovision hook: DriverInstanceActor.PostRestart -> ReprovisionRequested -> host re-init + re-subscribe (03/S6, S16)",
"status": "pending", "status": "done",
"blockedBy": [ "blockedBy": [
"T3" "T3"
] ]
@@ -1088,6 +1088,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied); Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied);
Receive<DriverInstanceActor.ApplyResult>(HandleApplyResult); Receive<DriverInstanceActor.ApplyResult>(HandleApplyResult);
Receive<RestartDriver>(HandleRestartDriver); Receive<RestartDriver>(HandleRestartDriver);
Receive<DriverInstanceActor.ReprovisionRequested>(HandleReprovisionRequested);
Receive<ReconnectDriver>(HandleReconnectDriver); Receive<ReconnectDriver>(HandleReconnectDriver);
Receive<RouteNodeWrite>(HandleRouteNodeWrite); Receive<RouteNodeWrite>(HandleRouteNodeWrite);
Receive<RouteNativeAlarmAck>(HandleRouteNativeAlarmAck); Receive<RouteNativeAlarmAck>(HandleRouteNativeAlarmAck);
@@ -1143,6 +1144,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied); Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied);
Receive<DriverInstanceActor.ApplyResult>(HandleApplyResult); Receive<DriverInstanceActor.ApplyResult>(HandleApplyResult);
Receive<RestartDriver>(HandleRestartDriver); Receive<RestartDriver>(HandleRestartDriver);
Receive<DriverInstanceActor.ReprovisionRequested>(HandleReprovisionRequested);
Receive<ReconnectDriver>(HandleReconnectDriver); Receive<ReconnectDriver>(HandleReconnectDriver);
Receive<RouteNodeWrite>(HandleRouteNodeWrite); Receive<RouteNodeWrite>(HandleRouteNodeWrite);
Receive<RouteNativeAlarmAck>(HandleRouteNativeAlarmAck); Receive<RouteNativeAlarmAck>(HandleRouteNativeAlarmAck);
@@ -1725,6 +1727,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Receive<GetDiagnostics>(HandleGetDiagnostics); Receive<GetDiagnostics>(HandleGetDiagnostics);
Receive<RetryConfigDbConnection>(_ => TryRecoverFromStale()); Receive<RetryConfigDbConnection>(_ => TryRecoverFromStale());
Receive<RestartDriver>(HandleRestartDriver); Receive<RestartDriver>(HandleRestartDriver);
Receive<DriverInstanceActor.ReprovisionRequested>(HandleReprovisionRequested);
Receive<ReconnectDriver>(HandleReconnectDriver); Receive<ReconnectDriver>(HandleReconnectDriver);
Receive<RedundancyStateChanged>(OnRedundancyStateChanged); Receive<RedundancyStateChanged>(OnRedundancyStateChanged);
// The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes). // The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes).
@@ -2991,6 +2994,40 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
SpawnChild(entry.Spec); SpawnChild(entry.Spec);
} }
/// <summary>
/// Re-provision a driver child that Akka's supervision just RESTARTED in place (archreview 03/S6,
/// 03/S16) — the third and least visible of the three incarnation routes.
/// </summary>
/// <remarks>
/// A restart recreates the actor object around the SAME driver instance (the Props closure captured
/// it) without going through <see cref="SpawnChild"/>, so neither the <c>InitializeRequested</c> nor
/// the desired-subscription send that a spawn performs ever happens — and the child's
/// <see cref="IActorRef"/> is unchanged, so <c>_children</c> needs no maintenance and the host has no
/// other notification that a new incarnation exists. Its <c>PreRestart</c> ran <c>PostStop</c>, which
/// called <c>ShutdownAsync</c> on the driver, so re-initialising is not optional: without it the
/// child sits in <c>Connecting</c> forever around a shut-down driver.
/// <para>
/// The dependency-consumer mux adapter is a SEPARATE child of this host and is unaffected by the
/// instance actor restarting, so it is deliberately not touched here.
/// </para>
/// </remarks>
/// <param name="msg">The restarted child's self-announcement.</param>
private void HandleReprovisionRequested(DriverInstanceActor.ReprovisionRequested msg)
{
// Already stopped (a restart racing a reconcile that removed the driver) — nothing to provision.
if (!_children.TryGetValue(msg.DriverInstanceId, out var entry)) return;
// A crash that Akka silently restarted is otherwise invisible to operators; name it.
_log.Warning(
"DriverHost {Node}: driver child {Id} was restarted by supervision after {Reason} — re-provisioning",
_localNode, msg.DriverInstanceId, msg.Reason);
// A stub child has no real driver to initialise (its Stubbed behaviour no-ops InitializeRequested
// anyway), but it still stores a desired set, so only the initialise is conditional.
if (!entry.Stubbed) entry.Actor.Tell(new DriverInstanceActor.InitializeRequested(entry.Spec.DriverConfig));
SendDesiredSubscriptions(entry.Actor, msg.DriverInstanceId);
}
private void HandleReconnectDriver(ReconnectDriver msg) private void HandleReconnectDriver(ReconnectDriver msg)
{ {
// DPS broadcast — only act if this node hosts the requested instance. // DPS broadcast — only act if this node hosts the requested instance.
@@ -42,6 +42,25 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
/// restored → Good) — independently of alarm transitions, since a comms-lost driver emits no alarm /// restored → Good) — independently of alarm transitions, since a comms-lost driver emits no alarm
/// events. Fire-and-forget, mirroring <see cref="DeltaApplied"/>.</summary> /// events. Fire-and-forget, mirroring <see cref="DeltaApplied"/>.</summary>
public sealed record ConnectivityChanged(string DriverInstanceId, bool Connected); public sealed record ConnectivityChanged(string DriverInstanceId, bool Connected);
/// <summary>
/// Sent to the parent from <see cref="PostRestart"/> so the host re-provisions THIS incarnation
/// (archreview 03/S6, 03/S16).
/// </summary>
/// <remarks>
/// A supervision restart recreates the actor object in place: the desired subscription sets are gone
/// (they are instance fields), and <c>PreRestart</c> has already run <see cref="PostStop"/> — which
/// called <see cref="IDriver.ShutdownAsync"/> on the driver object the Props closure re-uses. Nothing
/// else re-sends <see cref="InitializeRequested"/>: its only sender is
/// <c>DriverHostActor.SpawnChild</c>, and a restart does NOT respawn (the child's
/// <see cref="IActorRef"/> is unchanged, so the host is not otherwise notified that a new incarnation
/// exists at all). Without this the restarted child sits in <c>Connecting</c> forever around a
/// shut-down driver, with nothing subscribed — silently, since Akka logs the fault but the recovery
/// is invisible.
/// </remarks>
/// <param name="DriverInstanceId">The driver instance that was restarted.</param>
/// <param name="Reason">The type name of the exception that tripped supervision, for the host's log.</param>
public sealed record ReprovisionRequested(string DriverInstanceId, string Reason);
public sealed record ApplyDelta(string DriverConfigJson, CorrelationId Correlation); public sealed record ApplyDelta(string DriverConfigJson, CorrelationId Correlation);
public sealed record ApplyResult(bool Success, string? Reason, CorrelationId Correlation); public sealed record ApplyResult(bool Success, string? Reason, CorrelationId Correlation);
/// <summary> /// <summary>
@@ -375,6 +394,20 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
Timers.StartPeriodicTimer("health-poll", HealthPollTick.Instance, _healthPollInterval); Timers.StartPeriodicTimer("health-poll", HealthPollTick.Instance, _healthPollInterval);
} }
/// <inheritdoc />
/// <remarks>
/// Ask the host to re-provision this incarnation — see <see cref="ReprovisionRequested"/> for why a
/// supervision restart otherwise leaves a dead child. <c>base.PostRestart</c> runs first because it
/// runs <see cref="PreStart"/>, which re-attaches the rediscovery / host-status sources and restarts
/// the health timer; the reprovision is the part <see cref="PreStart"/> cannot do, since only the
/// host knows this driver's config and desired set.
/// </remarks>
protected override void PostRestart(Exception reason)
{
base.PostRestart(reason);
Context.Parent.Tell(new ReprovisionRequested(_driverInstanceId, reason.GetType().Name));
}
private void Stubbed() private void Stubbed()
{ {
// Stubbed drivers accept the standard message contracts but return deterministic // Stubbed drivers accept the standard message contracts but return deterministic