feat(mesh): node subscribers accept EventStream commands; _coordinatorOverride -> _ackRouter

Phase 2 Task 5. DriverHostActor and ScriptedAlarmHostActor now subscribe to the
node-local EventStream alongside their existing DPS topic subscriptions, so the
transport flag lives ENTIRELY on the publishing side -- exactly one of the two
ever publishes, and the switch can be flipped or reverted without touching a
single subscriber.

Note the alarm case is not symmetric: the DPS subscribe there still carries the
node-LOCAL publisher (OtOpcUaServerHostedService's OPC UA Part 9 method calls),
which never crosses the boundary and stays on DPS in both modes. Only the
central AdminUI leg moves.

Renamed DriverHostActor._coordinatorOverride to _ackRouter. Under ClusterClient
mode that ref is the NodeCommunicationActor, emphatically not a coordinator,
and the old name would send the next reader looking for one.

Adds two wiring tests, because the unit tests either side of this seam BOTH pass
with the subscription deleted -- the comm actor's test asserts a probe receives
the message, and the host tests drive the actor by direct Tell. Neither notices
a missing PreStart subscribe. Sabotage-verified: deleting either subscription
reddens exactly its own wiring test and nothing else.

Three things went wrong while writing those tests, all worth keeping:

- The first version raced. ActorOf returns before PreStart runs, and an
  EventStream.Publish with no subscriber is dropped silently -- no buffering, no
  dead letter. Fixed with a warm-up whose ack proves PreStart completed; the
  comment says why it is not ceremony.
- The alarm version was VACUOUS: it told the host directly INSIDE the assertion
  window alongside the relay, so the direct Tell alone produced the log the
  assertion waited for. Split into warm-up (outside) and relay (inside).
- The alarm test needs its own ActorSystem: the shared harness pins
  loglevel = WARNING and the only signal an unowned alarm gives is a Debug line.

Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
This commit is contained in:
Joseph Doherty
2026-07-22 12:19:29 -04:00
parent 915beec11a
commit 5cb0e72166
7 changed files with 212 additions and 23 deletions
@@ -86,7 +86,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
private readonly CommonsNodeId _localNode;
private readonly IActorRef? _coordinatorOverride;
private readonly IActorRef? _ackRouter;
private readonly IDriverFactory _driverFactory;
/// <summary>Builds the per-driver-instance <see cref="IDriverCapabilityInvoker"/> attached to each
@@ -328,7 +328,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
/// <summary>Creates props for a DriverHostActor with the specified dependencies.</summary>
/// <param name="dbFactory">Database context factory for configuration database access.</param>
/// <param name="localNode">The local cluster node identifier.</param>
/// <param name="coordinator">Optional coordinator actor reference for deployment coordination.</param>
/// <param name="ackRouter">Where ApplyAcks are sent. A direct coordinator handle in tests; under
/// <c>MeshTransport:Mode = ClusterClient</c> the node's <c>NodeCommunicationActor</c>, which relays
/// across the boundary. Null publishes on the <c>deployment-acks</c> DPS topic.</param>
/// <param name="driverFactory">Optional driver factory; defaults to null factory if not provided.</param>
/// <param name="localRoles">Optional set of roles assigned to the local node.</param>
/// <param name="dependencyMux">Optional actor reference for dependency multiplexing.</param>
@@ -364,7 +366,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
public static Props Props(
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
CommonsNodeId localNode,
IActorRef? coordinator = null,
IActorRef? ackRouter = null,
IDriverFactory? driverFactory = null,
IReadOnlySet<string>? localRoles = null,
IActorRef? dependencyMux = null,
@@ -387,7 +389,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
// the wrong dependency at runtime. New parameters go LAST in all three places — this
// signature, the constructor's, and this call — and nothing else moves.
Akka.Actor.Props.Create(() => new DriverHostActor(
dbFactory, localNode, coordinator, driverFactory, localRoles, dependencyMux, opcUaPublishActor,
dbFactory, localNode, ackRouter, driverFactory, localRoles, dependencyMux, opcUaPublishActor,
healthPublisher, virtualTagEvaluator, historyWriter, virtualTagHostOverride,
loggerFactory, scriptRootLogger, scriptedAlarmHostOverride, invokerFactory, driverMemberCountProvider,
deploymentArtifactCache, redundancyRoleView, replicationPeerHost));
@@ -395,7 +397,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
/// <summary>Initializes a new DriverHostActor with the specified dependencies.</summary>
/// <param name="dbFactory">Database context factory for configuration database access.</param>
/// <param name="localNode">The local cluster node identifier.</param>
/// <param name="coordinator">Optional coordinator actor reference for deployment coordination.</param>
/// <param name="ackRouter">Where ApplyAcks are sent. A direct coordinator handle in tests; under
/// <c>MeshTransport:Mode = ClusterClient</c> the node's <c>NodeCommunicationActor</c>, which relays
/// across the boundary. Null publishes on the <c>deployment-acks</c> DPS topic.</param>
/// <param name="driverFactory">Optional driver factory; defaults to null factory if not provided.</param>
/// <param name="localRoles">Optional set of roles assigned to the local node.</param>
/// <param name="dependencyMux">Optional actor reference for dependency multiplexing.</param>
@@ -426,7 +430,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
public DriverHostActor(
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
CommonsNodeId localNode,
IActorRef? coordinator,
IActorRef? ackRouter,
IDriverFactory? driverFactory = null,
IReadOnlySet<string>? localRoles = null,
IActorRef? dependencyMux = null,
@@ -449,7 +453,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
_replicationPeerHost = replicationPeerHost;
_dbFactory = dbFactory;
_localNode = localNode;
_coordinatorOverride = coordinator;
_ackRouter = ackRouter;
_driverFactory = driverFactory ?? NullDriverFactory.Instance;
_invokerFactory = invokerFactory ?? NullDriverCapabilityInvokerFactory.Instance;
_driverMemberCountProvider = driverMemberCountProvider;
@@ -483,6 +487,16 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
// uses so only the Primary services operator writes (the secondary keeps state warm for failover).
_mediator.Tell(
new Subscribe(ZB.MOM.WW.OtOpcUa.Runtime.OpcUa.OpcUaPublishActor.RedundancyStateTopic, Self));
// Per-cluster mesh Phase 2 dark switch. Under MeshTransport:Mode = ClusterClient these same
// commands arrive from NodeCommunicationActor on the node-local EventStream instead of the
// mesh-wide DPS topic. Subscribing to BOTH unconditionally is deliberate: exactly one of the
// two ever publishes, so the transport flag lives entirely on the publishing side and the
// switch can be flipped (or reverted) without touching any subscriber.
Context.System.EventStream.Subscribe(Self, typeof(DispatchDeployment));
Context.System.EventStream.Subscribe(Self, typeof(RestartDriver));
Context.System.EventStream.Subscribe(Self, typeof(ReconnectDriver));
// Spawn the VirtualTag host BEFORE Bootstrap so the bootstrap-restore path (which routes
// through PushDesiredSubscriptions and Tells ApplyVirtualTags) has a live host to target.
SpawnVirtualTagHost();
@@ -2410,15 +2424,20 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers
private void SendAck(DeploymentId deploymentId, ApplyAckOutcome outcome, string? failureReason, CorrelationId correlation)
{
var ack = new ApplyAck(deploymentId, _localNode, outcome, failureReason, correlation);
if (_coordinatorOverride is not null)
if (_ackRouter is not null)
{
_coordinatorOverride.Tell(ack);
// Either a direct coordinator handle (tests) or, under
// MeshTransport:Mode = ClusterClient, the NodeCommunicationActor that relays this
// across the boundary. Renamed from _coordinatorOverride in per-cluster mesh Phase 2:
// under ClusterClient mode this ref is emphatically NOT a coordinator, and the old
// name would send the next reader looking for one.
_ackRouter.Tell(ack);
}
else
{
// No direct coordinator handle — publish on the dedicated ACK topic. The coordinator
// singleton subscribes there in PreStart so the ACK reaches whichever admin node hosts
// it without an actor-path lookup.
// No router — publish on the dedicated ACK topic. The coordinator singleton subscribes
// there in PreStart so the ACK reaches whichever admin node hosts it without an
// actor-path lookup.
DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(DeploymentAcksTopic, ack));
}
}
@@ -528,6 +528,15 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
// RedundancyStateChanged and cache this node's role — OnEngineEmission gates the cluster-wide
// alerts publish to the Primary so a 2-node single-cluster deploy doesn't double-emit transitions.
_mediator.Tell(new Subscribe(OpcUaPublishActor.RedundancyStateTopic, Self));
// Per-cluster mesh Phase 2 dark switch. An AdminUI-originated ack/shelve arrives from
// NodeCommunicationActor on the node-local EventStream under
// MeshTransport:Mode = ClusterClient; the DPS subscribe above still carries the node-LOCAL
// publisher (OtOpcUaServerHostedService, the OPC UA Part 9 method calls), which never
// crosses the boundary and stays on DPS in both modes. Subscribing to both is safe: exactly
// one publisher feeds each path.
Context.System.EventStream.Subscribe(Self, typeof(AlarmCommand));
base.PreStart();
}
@@ -398,7 +398,7 @@ public static class ServiceCollectionExtensions
registry.Register<PeerProbeSupervisorKey>(peerProbes);
var driverHost = system.ActorOf(
DriverHostActor.Props(dbFactory, roleInfo.LocalNode, coordinator: null,
DriverHostActor.Props(dbFactory, roleInfo.LocalNode, ackRouter: null,
driverFactory: driverFactory, localRoles: roleInfo.LocalRoles,
dependencyMux: mux,
opcUaPublishActor: publishActor,