fix(runtime): re-provision a driver child after a supervision restart (archreview 03/S6, S16)
This commit is contained in:
@@ -595,3 +595,13 @@ discipline for this assembly).
|
||||
works: with T3 reverted this test fails on the *positive control*, not on the absence.
|
||||
**Not covered, and worth a reviewer's eye:** the `_desiredByDriver` union's `_children.Keys` arm
|
||||
(a running child absent from BOTH ref maps) is not reachable through this seeder and is untested.
|
||||
|
||||
### T6 — supervision reprovision hook
|
||||
|
||||
`DriverInstanceActor.PostRestart` (calling `base.PostRestart(reason)` first, so `PreStart` re-attaches
|
||||
the rediscovery / host-status sources and restarts the health timer) → `ReprovisionRequested` →
|
||||
`DriverHostActor.HandleReprovisionRequested`, registered in **all three** `Receive` blocks that already
|
||||
accept `RestartDriver`. The falsification is T7, below.
|
||||
|
||||
Gate: `--filter "…~DriverHostActor|…~DriverInstanceActor"` → **183/183 passed** (179 before this task
|
||||
plus T5's four), no test moved.
|
||||
|
||||
@@ -40,7 +40,7 @@
|
||||
{
|
||||
"id": "T6",
|
||||
"subject": "Supervision reprovision hook: DriverInstanceActor.PostRestart -> ReprovisionRequested -> host re-init + re-subscribe (03/S6, S16)",
|
||||
"status": "pending",
|
||||
"status": "done",
|
||||
"blockedBy": [
|
||||
"T3"
|
||||
]
|
||||
|
||||
@@ -1088,6 +1088,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied);
|
||||
Receive<DriverInstanceActor.ApplyResult>(HandleApplyResult);
|
||||
Receive<RestartDriver>(HandleRestartDriver);
|
||||
Receive<DriverInstanceActor.ReprovisionRequested>(HandleReprovisionRequested);
|
||||
Receive<ReconnectDriver>(HandleReconnectDriver);
|
||||
Receive<RouteNodeWrite>(HandleRouteNodeWrite);
|
||||
Receive<RouteNativeAlarmAck>(HandleRouteNativeAlarmAck);
|
||||
@@ -1143,6 +1144,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
Receive<DriverInstanceActor.DeltaApplied>(HandleDeltaApplied);
|
||||
Receive<DriverInstanceActor.ApplyResult>(HandleApplyResult);
|
||||
Receive<RestartDriver>(HandleRestartDriver);
|
||||
Receive<DriverInstanceActor.ReprovisionRequested>(HandleReprovisionRequested);
|
||||
Receive<ReconnectDriver>(HandleReconnectDriver);
|
||||
Receive<RouteNodeWrite>(HandleRouteNodeWrite);
|
||||
Receive<RouteNativeAlarmAck>(HandleRouteNativeAlarmAck);
|
||||
@@ -1725,6 +1727,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
Receive<GetDiagnostics>(HandleGetDiagnostics);
|
||||
Receive<RetryConfigDbConnection>(_ => TryRecoverFromStale());
|
||||
Receive<RestartDriver>(HandleRestartDriver);
|
||||
Receive<DriverInstanceActor.ReprovisionRequested>(HandleReprovisionRequested);
|
||||
Receive<ReconnectDriver>(HandleReconnectDriver);
|
||||
Receive<RedundancyStateChanged>(OnRedundancyStateChanged);
|
||||
// The redundancy-state topic also carries OpcUaProbeResult (OpcUaPublishActor peer-probes).
|
||||
@@ -2991,6 +2994,40 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
SpawnChild(entry.Spec);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Re-provision a driver child that Akka's supervision just RESTARTED in place (archreview 03/S6,
|
||||
/// 03/S16) — the third and least visible of the three incarnation routes.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A restart recreates the actor object around the SAME driver instance (the Props closure captured
|
||||
/// it) without going through <see cref="SpawnChild"/>, so neither the <c>InitializeRequested</c> nor
|
||||
/// the desired-subscription send that a spawn performs ever happens — and the child's
|
||||
/// <see cref="IActorRef"/> is unchanged, so <c>_children</c> needs no maintenance and the host has no
|
||||
/// other notification that a new incarnation exists. Its <c>PreRestart</c> ran <c>PostStop</c>, which
|
||||
/// called <c>ShutdownAsync</c> on the driver, so re-initialising is not optional: without it the
|
||||
/// child sits in <c>Connecting</c> forever around a shut-down driver.
|
||||
/// <para>
|
||||
/// The dependency-consumer mux adapter is a SEPARATE child of this host and is unaffected by the
|
||||
/// instance actor restarting, so it is deliberately not touched here.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="msg">The restarted child's self-announcement.</param>
|
||||
private void HandleReprovisionRequested(DriverInstanceActor.ReprovisionRequested msg)
|
||||
{
|
||||
// Already stopped (a restart racing a reconcile that removed the driver) — nothing to provision.
|
||||
if (!_children.TryGetValue(msg.DriverInstanceId, out var entry)) return;
|
||||
|
||||
// A crash that Akka silently restarted is otherwise invisible to operators; name it.
|
||||
_log.Warning(
|
||||
"DriverHost {Node}: driver child {Id} was restarted by supervision after {Reason} — re-provisioning",
|
||||
_localNode, msg.DriverInstanceId, msg.Reason);
|
||||
|
||||
// A stub child has no real driver to initialise (its Stubbed behaviour no-ops InitializeRequested
|
||||
// anyway), but it still stores a desired set, so only the initialise is conditional.
|
||||
if (!entry.Stubbed) entry.Actor.Tell(new DriverInstanceActor.InitializeRequested(entry.Spec.DriverConfig));
|
||||
SendDesiredSubscriptions(entry.Actor, msg.DriverInstanceId);
|
||||
}
|
||||
|
||||
private void HandleReconnectDriver(ReconnectDriver msg)
|
||||
{
|
||||
// DPS broadcast — only act if this node hosts the requested instance.
|
||||
|
||||
@@ -42,6 +42,25 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
/// restored → Good) — independently of alarm transitions, since a comms-lost driver emits no alarm
|
||||
/// events. Fire-and-forget, mirroring <see cref="DeltaApplied"/>.</summary>
|
||||
public sealed record ConnectivityChanged(string DriverInstanceId, bool Connected);
|
||||
|
||||
/// <summary>
|
||||
/// Sent to the parent from <see cref="PostRestart"/> so the host re-provisions THIS incarnation
|
||||
/// (archreview 03/S6, 03/S16).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A supervision restart recreates the actor object in place: the desired subscription sets are gone
|
||||
/// (they are instance fields), and <c>PreRestart</c> has already run <see cref="PostStop"/> — which
|
||||
/// called <see cref="IDriver.ShutdownAsync"/> on the driver object the Props closure re-uses. Nothing
|
||||
/// else re-sends <see cref="InitializeRequested"/>: its only sender is
|
||||
/// <c>DriverHostActor.SpawnChild</c>, and a restart does NOT respawn (the child's
|
||||
/// <see cref="IActorRef"/> is unchanged, so the host is not otherwise notified that a new incarnation
|
||||
/// exists at all). Without this the restarted child sits in <c>Connecting</c> forever around a
|
||||
/// shut-down driver, with nothing subscribed — silently, since Akka logs the fault but the recovery
|
||||
/// is invisible.
|
||||
/// </remarks>
|
||||
/// <param name="DriverInstanceId">The driver instance that was restarted.</param>
|
||||
/// <param name="Reason">The type name of the exception that tripped supervision, for the host's log.</param>
|
||||
public sealed record ReprovisionRequested(string DriverInstanceId, string Reason);
|
||||
public sealed record ApplyDelta(string DriverConfigJson, CorrelationId Correlation);
|
||||
public sealed record ApplyResult(bool Success, string? Reason, CorrelationId Correlation);
|
||||
/// <summary>
|
||||
@@ -375,6 +394,20 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
Timers.StartPeriodicTimer("health-poll", HealthPollTick.Instance, _healthPollInterval);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Ask the host to re-provision this incarnation — see <see cref="ReprovisionRequested"/> for why a
|
||||
/// supervision restart otherwise leaves a dead child. <c>base.PostRestart</c> runs first because it
|
||||
/// runs <see cref="PreStart"/>, which re-attaches the rediscovery / host-status sources and restarts
|
||||
/// the health timer; the reprovision is the part <see cref="PreStart"/> cannot do, since only the
|
||||
/// host knows this driver's config and desired set.
|
||||
/// </remarks>
|
||||
protected override void PostRestart(Exception reason)
|
||||
{
|
||||
base.PostRestart(reason);
|
||||
Context.Parent.Tell(new ReprovisionRequested(_driverInstanceId, reason.GetType().Name));
|
||||
}
|
||||
|
||||
private void Stubbed()
|
||||
{
|
||||
// Stubbed drivers accept the standard message contracts but return deterministic
|
||||
|
||||
Reference in New Issue
Block a user