fix(site-runtime): watch every Instance Actor; unexpected termination evicts the dead ref from routing (S6)

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/DeploymentManagerActor.cs

@@ -526,14 +526,21 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
     }
 
     /// <summary>
-    /// Recreates an Instance Actor once its predecessor has fully terminated during a
+    /// Handles an Instance Actor's <see cref="Terminated"/> signal. Two cases:
-    /// redeployment, draining the buffered <see cref="DeployInstanceCommand"/>.
+    /// <list type="number">
+    /// <item>The child was stopped as part of a redeployment — recreate it from the
+    /// buffered <see cref="DeployInstanceCommand"/>.</item>
+    /// <item>The child terminated OUTSIDE the redeploy flow — an init failure or crash.
+    /// Evict the now-dead ref from the routing map so the singleton never routes to a
+    /// stopped actor (S6).</item>
+    /// </list>
     /// </summary>
     private void HandleTerminated(Terminated terminated)
     {
-        if (!_pendingRedeploys.Remove(terminated.ActorRef, out var pending))
+        // Case 1 — redeployment drain: the predecessor of a buffered redeploy has
-            return;
+        // fully stopped and freed its actor name.
-
+        if (_pendingRedeploys.Remove(terminated.ActorRef, out var pending))
+        {
             // Drop the name-keyed shadow now that the predecessor has
             // fully terminated and its actor name is free. Any deploy arriving after
             // this point sees neither _instanceActors[name] (cleared when we stopped
@@ -542,6 +549,25 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
             _terminatingActorsByName.Remove(pending.Command.InstanceUniqueName);
 
             ApplyDeployment(pending.Command, pending.OriginalSender, isRedeploy: true);
+            return;
+        }
+
+        // Case 2 — S6 fallback: an Instance Actor terminated unexpectedly (init
+        // failure or crash). Expected stops (disable/delete/redeploy/persistence
+        // rollback) all remove or replace the map entry BEFORE the child stops, so
+        // the equality guard is false for them and this branch is inert. Only a dead
+        // ref still mapped to this exact actor reaches here.
+        var name = terminated.ActorRef.Path.Name;
+        if (_instanceActors.TryGetValue(name, out var mapped) && mapped.Equals(terminated.ActorRef))
+        {
+            _instanceActors.Remove(name);
+            UpdateInstanceCounts();
+            _logger.LogError(
+                "Instance Actor {Instance} terminated unexpectedly (init failure or crash) — removed from routing",
+                name);
+            LogDeploymentEvent("Error", name,
+                $"Instance {name} terminated unexpectedly (init failure or crash) — removed from routing");
+        }
     }
 
     /// <summary>
@@ -1718,6 +1744,11 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
 
         var actorRef = Context.ActorOf(props, instanceName);
         _instanceActors[instanceName] = actorRef;
+        // Watch every Instance Actor so an unexpected termination (init failure or
+        // crash) delivers a Terminated signal that evicts the dead ref from routing
+        // (S6). Redeploy already double-watches; a double Watch is idempotent in
+        // Akka.NET (a single Terminated is delivered).
+        Context.Watch(actorRef);
 
         _logger.LogDebug("Created Instance Actor for {Instance}", instanceName);
     }

tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/DeploymentManagerRedeployTests.cs

@@ -10,6 +10,7 @@ using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
 using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
 using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
 using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
+using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
 using System.Text.Json;
 
 namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
@@ -45,7 +46,8 @@ public class DeploymentManagerRedeployTests : TestKit, IDisposable
         try { File.Delete(_dbFile); } catch { /* cleanup */ }
     }
 
-    private IActorRef CreateDeploymentManager(ISiteHealthCollector? healthCollector = null)
+    private IActorRef CreateDeploymentManager(
+        ISiteHealthCollector? healthCollector = null, IServiceProvider? serviceProvider = null)
     {
         return ActorOf(Props.Create(() => new DeploymentManagerActor(
             _storage,
@@ -57,9 +59,31 @@ public class DeploymentManagerRedeployTests : TestKit, IDisposable
             null,
             null,
             healthCollector,
-            null)));
+            serviceProvider)));
     }
 
+    /// <summary>
+    /// Builds a config that deserializes and compiles cleanly (so it passes the deploy
+    /// compile gate) but whose script's canonical name contains a space — illegal as an
+    /// Akka actor name, so <c>Context.ActorOf(props, "script-dies on init")</c> throws
+    /// <c>InvalidActorNameException</c> during the Instance Actor's PreStart. The failure
+    /// surfaces as an <c>ActorInitializationException</c> the DM decider stops — the exact
+    /// "Instance Actor dies during init" S6 scenario.
+    /// </summary>
+    private static string CtorThrowingConfig(string instanceName) =>
+        JsonSerializer.Serialize(new FlattenedConfiguration
+        {
+            InstanceUniqueName = instanceName,
+            Attributes =
+            [
+                new ResolvedAttribute { CanonicalName = "TestAttr", Value = "1", DataType = "Int32" }
+            ],
+            Scripts =
+            [
+                new ResolvedScript { CanonicalName = "dies on init", Code = "return 1;" }
+            ]
+        });
+
     /// <summary>
     /// Minimal fake that records the most recent deployed-instance count.
     /// </summary>
@@ -311,4 +335,29 @@ public class DeploymentManagerRedeployTests : TestKit, IDisposable
         // not a new instance, so the in-memory counter must not drift upward.
         Assert.Equal(1, health.LastDeployedCount);
     }
+
+    // ── S6: an Instance Actor that dies during init must not linger as a dead ref ──
+
+    [Fact]
+    public async Task InstanceActorInitFailure_OnStartup_EvictsDeadRefAndLogsSiteEvent()
+    {
+        // A stored config that compiles cleanly but whose Instance Actor cannot
+        // initialize (illegal child actor name → ActorInitializationException → Stop).
+        // The DM watches every Instance Actor, so on the unexpected Termination it must
+        // evict the dead ref from the routing map and log a `deployment` Error event
+        // rather than leaving a routing entry that points at a stopped actor (S6).
+        await _storage.StoreDeployedConfigAsync(
+            "poison", CtorThrowingConfig("poison"), "d1", "h1", true);
+
+        var siteLog = new FakeSiteEventLogger();
+        CreateDeploymentManager(serviceProvider: new SingleServiceProvider(siteLog));
+
+        AwaitAssert(() =>
+        {
+            Assert.Contains(siteLog.OfType("deployment"), r =>
+                r.Severity == "Error" &&
+                r.InstanceId == "poison" &&
+                r.Message.Contains("terminated unexpectedly", StringComparison.OrdinalIgnoreCase));
+        }, TimeSpan.FromSeconds(10));
+    }
 }