fix(site-runtime): watch every Instance Actor; unexpected termination evicts the dead ref from routing (S6)

This commit is contained in:
Joseph Doherty
2026-07-09 00:20:12 -04:00
parent 2dabbbdde0
commit 9343b4f30f
2 changed files with 93 additions and 13 deletions
@@ -526,22 +526,48 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
}
/// <summary>
/// Recreates an Instance Actor once its predecessor has fully terminated during a
/// redeployment, draining the buffered <see cref="DeployInstanceCommand"/>.
/// Handles an Instance Actor's <see cref="Terminated"/> signal. Two cases:
/// <list type="number">
/// <item>The child was stopped as part of a redeployment — recreate it from the
/// buffered <see cref="DeployInstanceCommand"/>.</item>
/// <item>The child terminated OUTSIDE the redeploy flow — an init failure or crash.
/// Evict the now-dead ref from the routing map so the singleton never routes to a
/// stopped actor (S6).</item>
/// </list>
/// </summary>
private void HandleTerminated(Terminated terminated)
{
if (!_pendingRedeploys.Remove(terminated.ActorRef, out var pending))
// Case 1 — redeployment drain: the predecessor of a buffered redeploy has
// fully stopped and freed its actor name.
if (_pendingRedeploys.Remove(terminated.ActorRef, out var pending))
{
// Drop the name-keyed shadow now that the predecessor has
// fully terminated and its actor name is free. Any deploy arriving after
// this point sees neither _instanceActors[name] (cleared when we stopped
// the predecessor) nor _terminatingActorsByName[name] (cleared here), so
// ApplyDeployment + Context.ActorOf below safely reuses the name.
_terminatingActorsByName.Remove(pending.Command.InstanceUniqueName);
ApplyDeployment(pending.Command, pending.OriginalSender, isRedeploy: true);
return;
}
// Drop the name-keyed shadow now that the predecessor has
// fully terminated and its actor name is free. Any deploy arriving after
// this point sees neither _instanceActors[name] (cleared when we stopped
// the predecessor) nor _terminatingActorsByName[name] (cleared here), so
// ApplyDeployment + Context.ActorOf below safely reuses the name.
_terminatingActorsByName.Remove(pending.Command.InstanceUniqueName);
ApplyDeployment(pending.Command, pending.OriginalSender, isRedeploy: true);
// Case 2 — S6 fallback: an Instance Actor terminated unexpectedly (init
// failure or crash). Expected stops (disable/delete/redeploy/persistence
// rollback) all remove or replace the map entry BEFORE the child stops, so
// the equality guard is false for them and this branch is inert. Only a dead
// ref still mapped to this exact actor reaches here.
var name = terminated.ActorRef.Path.Name;
if (_instanceActors.TryGetValue(name, out var mapped) && mapped.Equals(terminated.ActorRef))
{
_instanceActors.Remove(name);
UpdateInstanceCounts();
_logger.LogError(
"Instance Actor {Instance} terminated unexpectedly (init failure or crash) — removed from routing",
name);
LogDeploymentEvent("Error", name,
$"Instance {name} terminated unexpectedly (init failure or crash) — removed from routing");
}
}
/// <summary>
@@ -1718,6 +1744,11 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
var actorRef = Context.ActorOf(props, instanceName);
_instanceActors[instanceName] = actorRef;
// Watch every Instance Actor so an unexpected termination (init failure or
// crash) delivers a Terminated signal that evicts the dead ref from routing
// (S6). Redeploy already double-watches; a double Watch is idempotent in
// Akka.NET (a single Terminated is delivered).
Context.Watch(actorRef);
_logger.LogDebug("Created Instance Actor for {Instance}", instanceName);
}