feat(mesh): node subscribers accept EventStream commands; _coordinatorOverride -> _ackRouter

Phase 2 Task 5. DriverHostActor and ScriptedAlarmHostActor now subscribe to the
node-local EventStream alongside their existing DPS topic subscriptions, so the
transport flag lives ENTIRELY on the publishing side -- exactly one of the two
ever publishes, and the switch can be flipped or reverted without touching a
single subscriber.

Note the alarm case is not symmetric: the DPS subscribe there still carries the
node-LOCAL publisher (OtOpcUaServerHostedService's OPC UA Part 9 method calls),
which never crosses the boundary and stays on DPS in both modes. Only the
central AdminUI leg moves.

Renamed DriverHostActor._coordinatorOverride to _ackRouter. Under ClusterClient
mode that ref is the NodeCommunicationActor, emphatically not a coordinator,
and the old name would send the next reader looking for one.

Adds two wiring tests, because the unit tests either side of this seam BOTH pass
with the subscription deleted -- the comm actor's test asserts a probe receives
the message, and the host tests drive the actor by direct Tell. Neither notices
a missing PreStart subscribe. Sabotage-verified: deleting either subscription
reddens exactly its own wiring test and nothing else.

Three things went wrong while writing those tests, all worth keeping:

- The first version raced. ActorOf returns before PreStart runs, and an
  EventStream.Publish with no subscriber is dropped silently -- no buffering, no
  dead letter. Fixed with a warm-up whose ack proves PreStart completed; the
  comment says why it is not ceremony.
- The alarm version was VACUOUS: it told the host directly INSIDE the assertion
  window alongside the relay, so the direct Tell alone produced the log the
  assertion waited for. Split into warm-up (outside) and relay (inside).
- The alarm test needs its own ActorSystem: the shared harness pins
  loglevel = WARNING and the only signal an unowned alarm gives is a Debug line.

Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW

tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Communication/EventStreamCommandDeliveryTests.cs

@@ -0,0 +1,161 @@
+using Akka.Actor;
+using Akka.TestKit.Xunit2;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
+using ZB.MOM.WW.OtOpcUa.Commons.Types;
+using ZB.MOM.WW.OtOpcUa.Runtime.Communication;
+using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
+using Serilog;
+using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
+using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
+using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
+using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
+using ZB.MOM.WW.OtOpcUa.Core.Scripting;
+
+namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Communication;
+
+/// <summary>
+///     Proves the Phase 2 node-side leg joins up: a command handed to
+///     <see cref="NodeCommunicationActor"/> reaches the <b>real</b> <see cref="DriverHostActor"/>,
+///     not just a test probe.
+/// </summary>
+/// <remarks>
+///     <para>
+///         The unit tests either side of this seam both pass with the wiring broken.
+///         <c>NodeCommunicationActorTests</c> asserts a probe subscribed to the EventStream receives
+///         the message; the driver-host tests drive the actor by direct <c>Tell</c>. Neither notices
+///         if <c>DriverHostActor.PreStart</c> never subscribes. That gap is why this file exists.
+///     </para>
+///     <para>
+///         <b>Ordering matters, and it bit this test first time out.</b> <c>ActorOf</c> returns
+///         before <c>PreStart</c> has run, and an <c>EventStream.Publish</c> with no subscriber is
+///         dropped silently — there is no buffering and no dead-letter. Relaying immediately after
+///         spawning therefore fails intermittently. The warm-up below is not ceremony: it is the
+///         only available proof that <c>PreStart</c> has completed. (In production the driver host
+///         is spawned at startup, long before any deployment, so the same race does not arise.)
+///     </para>
+/// </remarks>
+public class EventStreamCommandDeliveryTests : RuntimeActorTestBase
+{
+    private static DispatchDeployment NewDispatch() => new(
+        new DeploymentId(Guid.NewGuid()),
+        new RevisionHash("rev-does-not-exist"),
+        new CorrelationId(Guid.NewGuid()));
+
+    [Fact]
+    public void A_dispatch_relayed_by_the_comm_actor_reaches_the_real_driver_host()
+    {
+        var ackRouter = CreateTestProbe();
+        var driverHost = Sys.ActorOf(
+            DriverHostActor.Props(NewInMemoryDbFactory(), new NodeId("127.0.0.1:4053"), ackRouter: ackRouter.Ref),
+            "driver-host");
+        var comm = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null), "node-communication");
+
+        // Warm-up: a direct Tell whose ack proves PreStart has completed, and with it the
+        // EventStream subscription. Without this the relay below races the subscription.
+        var warmup = NewDispatch();
+        driverHost.Tell(warmup);
+        ackRouter.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(15)).DeploymentId.ShouldBe(warmup.DeploymentId);
+
+        // Now the real assertion — relayed exactly as central would send it. The comm actor only
+        // publishes node-locally, so the host must be subscribed for this to arrive at all.
+        var relayed = NewDispatch();
+        comm.Tell(relayed);
+
+        // The deployment id does not exist, so the host answers Failed. That is the stronger
+        // assertion, not a weaker one: an ack at all proves the command was received and handled,
+        // and only PreStart's EventStream subscription can have delivered it.
+        ackRouter.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(15)).DeploymentId.ShouldBe(relayed.DeploymentId);
+    }
+
+    [Fact]
+    public void CONTROL_the_driver_host_acks_a_direct_tell()
+    {
+        // Falsifiability control. If the test above fails, this one says which half broke: green
+        // here means the host is healthy and the EventStream relay is at fault; red here means the
+        // host would not have acked either way and the relay is not implicated.
+        var ackRouter = CreateTestProbe();
+        var driverHost = Sys.ActorOf(
+            DriverHostActor.Props(NewInMemoryDbFactory(), new NodeId("127.0.0.1:4053"), ackRouter: ackRouter.Ref),
+            "driver-host-control");
+
+        var dispatch = NewDispatch();
+        driverHost.Tell(dispatch);
+
+        ackRouter.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(15)).DeploymentId.ShouldBe(dispatch.DeploymentId);
+    }
+
+}
+
+/// <summary>
+///     The alarm leg of the same wiring proof, in its own ActorSystem because the shared
+///     <see cref="RuntimeActorTestBase"/> harness pins <c>loglevel = WARNING</c> and the only
+///     observable signal <c>ScriptedAlarmHostActor</c> gives for an unowned alarm is a Debug line.
+/// </summary>
+public class AlarmCommandEventStreamDeliveryTests : TestKit
+{
+    private static string DebugLevelHocon => @"
+akka {
+    loglevel = ""DEBUG""
+    extensions = [
+        ""Akka.Cluster.Tools.PublishSubscribe.DistributedPubSubExtensionProvider, Akka.Cluster.Tools""
+    ]
+    actor {
+        provider = ""Akka.Cluster.ClusterActorRefProvider, Akka.Cluster""
+    }
+    remote.dot-netty.tcp {
+        hostname = ""127.0.0.1""
+        port = 0
+    }
+    cluster {
+        seed-nodes = []
+        roles = [""driver""]
+        min-nr-of-members = 1
+        run-coordinated-shutdown-when-down = off
+    }
+}";
+
+    public AlarmCommandEventStreamDeliveryTests()
+        : base(DebugLevelHocon)
+    {
+        var cluster = Akka.Cluster.Cluster.Get(Sys);
+        cluster.Join(cluster.SelfAddress);
+        AwaitCondition(
+            () => cluster.State.Members.Any(m => m.Status == Akka.Cluster.MemberStatus.Up),
+            TimeSpan.FromSeconds(5));
+    }
+
+    [Fact]
+    public void An_alarm_command_relayed_by_the_comm_actor_reaches_the_real_scripted_alarm_host()
+    {
+        // Needed for the same reason as the deploy leg: the comm actor's unit test only proves a
+        // probe receives it, and the alarm-host tests drive the actor by direct Tell. Neither
+        // notices a missing PreStart subscription.
+        var upstream = new DependencyMuxTagUpstreamSource();
+        var logger = new LoggerConfiguration().CreateLogger();
+        var engine = new ScriptedAlarmEngine(
+            upstream, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger);
+
+        var host = Sys.ActorOf(
+            ScriptedAlarmHostActor.Props(
+                CreateTestProbe().Ref, CreateTestProbe().Ref, upstream, engine, new NodeId("node-A")),
+            "scripted-alarm-host");
+        var comm = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null), "node-communication");
+
+        var cmd = new AlarmCommand("alarm-unowned", "Acknowledge", "alice", "c", null);
+
+        // Warm-up, OUTSIDE the real assertion: a direct Tell whose debug line proves PreStart has
+        // completed and the EventStream subscription is registered. An earlier draft did the direct
+        // Tell INSIDE the assertion window alongside the relay, which made the test vacuous — the
+        // direct Tell alone produced the log the assertion was waiting for.
+        EventFilter.Debug(contains: "ignoring AlarmCommand")
+            .ExpectOne(TimeSpan.FromSeconds(15), () => host.Tell(cmd));
+
+        // The real assertion: relayed ONLY through the comm actor, which publishes node-locally.
+        // The host owns no alarms, so it logs and stops — still a positive delivery signal, because
+        // that line is only reached if the message arrived.
+        EventFilter.Debug(contains: "ignoring AlarmCommand")
+            .ExpectOne(TimeSpan.FromSeconds(15), () => comm.Tell(cmd));
+    }
+}

tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorBootFromCacheTests.cs

@@ -41,7 +41,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
         var cache = new StubArtifactCache(cached);
 
         var actor = Sys.ActorOf(DriverHostActor.Props(
-            new ThrowingDbFactory(), TestNode, coordinator: null,
+            new ThrowingDbFactory(), TestNode, ackRouter: null,
             localRoles: new HashSet<string> { "driver" },
             deploymentArtifactCache: cache));
 
@@ -70,7 +70,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
         var publishProbe = CreateTestProbe();
 
         Sys.ActorOf(DriverHostActor.Props(
-            new ThrowingDbFactory(), TestNode, coordinator: null,
+            new ThrowingDbFactory(), TestNode, ackRouter: null,
             localRoles: new HashSet<string> { "driver" },
             opcUaPublishActor: publishProbe.Ref,
             deploymentArtifactCache: new StubArtifactCache(cached)));
@@ -85,7 +85,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
     {
         // The pre-cache behaviour, unchanged. A cache miss must not invent a configuration.
         var actor = Sys.ActorOf(DriverHostActor.Props(
-            new ThrowingDbFactory(), TestNode, coordinator: null,
+            new ThrowingDbFactory(), TestNode, ackRouter: null,
             localRoles: new HashSet<string> { "driver" },
             deploymentArtifactCache: new StubArtifactCache(current: null)));
 
@@ -100,7 +100,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
     {
         // Admin-only graphs and older harnesses pass null; behaviour is identical to a cache miss.
         var actor = Sys.ActorOf(DriverHostActor.Props(
-            new ThrowingDbFactory(), TestNode, coordinator: null,
+            new ThrowingDbFactory(), TestNode, ackRouter: null,
             localRoles: new HashSet<string> { "driver" }));
 
         var snapshot = AskDiagnostics(actor);
@@ -118,7 +118,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
             DeploymentId.NewId().ToString(), CachedRev.Value, EmptyArtifact(), DateTimeOffset.UtcNow));
 
         var actor = Sys.ActorOf(DriverHostActor.Props(
-            NewInMemoryDbFactory(), TestNode, coordinator: null,
+            NewInMemoryDbFactory(), TestNode, ackRouter: null,
             localRoles: new HashSet<string> { "driver" },
             deploymentArtifactCache: cache));
 
@@ -140,7 +140,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
             deploymentId.ToString(), CachedRev.Value, EmptyArtifact(), DateTimeOffset.UtcNow));
 
         var actor = Sys.ActorOf(DriverHostActor.Props(
-            db, TestNode, coordinator: null,
+            db, TestNode, ackRouter: null,
             localRoles: new HashSet<string> { "driver" },
             deploymentArtifactCache: cache));
 
@@ -156,7 +156,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
     {
         // A cache fault must leave the node exactly where it would have been without a cache.
         var actor = Sys.ActorOf(DriverHostActor.Props(
-            new ThrowingDbFactory(), TestNode, coordinator: null,
+            new ThrowingDbFactory(), TestNode, ackRouter: null,
             localRoles: new HashSet<string> { "driver" },
             deploymentArtifactCache: new ThrowingReadCache()));
 
@@ -183,7 +183,7 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
         var cache = new StubArtifactCache(current: null);
 
         var actor = Sys.ActorOf(DriverHostActor.Props(
-            db, TestNode, coordinator: null,
+            db, TestNode, ackRouter: null,
             localRoles: new HashSet<string> { "driver" },
             deploymentArtifactCache: cache));
 

tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRoleViewTests.cs

@@ -183,7 +183,7 @@ public sealed class DriverHostActorRoleViewTests : RuntimeActorTestBase
         Sys.ActorOf(DriverHostActor.Props(
             NewInMemoryDbFactory(),
             TestNode,
-            coordinator: null,
+            ackRouter: null,
             driverMemberCountProvider: () => driverMemberCount,
             redundancyRoleView: view,
             replicationPeerHost: PeerHost));

tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxActorTests.cs

@@ -139,7 +139,7 @@ public sealed class DependencyMuxActorTests : RuntimeActorTestBase
         var hostActor = Sys.ActorOf(DriverHostActor.Props(
             dbFactory: NewInMemoryDbFactory(),
             localNode: ZB.MOM.WW.OtOpcUa.Commons.Types.NodeId.Parse("host-1"),
-            coordinator: hostProbe.Ref,
+            ackRouter: hostProbe.Ref,
             dependencyMux: mux));
 
         // Tell the host an AttributeValuePublished — it should fan out to the mux + subscriber.