From 64774652b733a22becc5f185ef9964261d7edf88 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 6 Aug 2026 05:14:12 -0400 Subject: [PATCH 01/12] feat(mesh): NodeCommandService on the telemetry listener --- .../Grpc/NodeCommandGrpcService.cs | 110 +++++++++++ src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs | 9 + .../AssemblyInfo.cs | 3 +- .../Grpc/NodeCommandGrpcServiceTests.cs | 186 ++++++++++++++++++ .../ZB.MOM.WW.OtOpcUa.Host.Tests.csproj | 3 + 5 files changed, 310 insertions(+), 1 deletion(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/Grpc/NodeCommandGrpcService.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Grpc/NodeCommandGrpcServiceTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Grpc/NodeCommandGrpcService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Grpc/NodeCommandGrpcService.cs new file mode 100644 index 00000000..473c4366 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Grpc/NodeCommandGrpcService.cs @@ -0,0 +1,110 @@ +using Akka.Actor; +using Akka.Hosting; +using Grpc.Core; +using Microsoft.Extensions.Logging; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; +using ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1; +using ZB.MOM.WW.OtOpcUa.Runtime; +// The generated service container is itself named NodeCommandService; alias its nested server base so +// this impl can keep the natural name without colliding with the generated type. +using GeneratedServiceBase = + ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1.NodeCommandService.NodeCommandServiceBase; +// Akka.Actor (imported for IActorRef.Tell) declares a Status of its own; only the gRPC one is used here. +using Status = Grpc.Core.Status; + +namespace ZB.MOM.WW.OtOpcUa.Host.Grpc; + +/// +/// Node-side command ingress (ClusterClient→gRPC migration Phase 1). Central dials a driver node +/// and calls ; the envelope is decoded back into the same CLR record the +/// ClusterClient receptionist used to deliver and handed to this node's +/// NodeCommunicationActor, which re-emits it on the node's EventStream exactly as +/// before. Nothing downstream of that actor can tell which transport carried the command. +/// +/// +/// +/// One-way, verbatim from the transport it replaces. ExecuteAck means "decoded and +/// handed to the comm actor" — not applied, and not acknowledged by any EventStream subscriber. +/// The Tell is deliberately not an Ask: no migrated command has a reply, and +/// waiting for one would invent a semantic central does not have. +/// +/// +/// Decode runs BEFORE the registry lookup, and the two failures are graded differently. +/// An undecodable envelope is — permanent, and it will +/// never decode on a retry. A call arriving before AddOtOpcUaRuntime has spawned and +/// registered the comm actor is — the house readiness +/// convention, transient, and correct to retry. Checking readiness first would report a +/// permanently malformed command as a readiness problem and invite an unbounded retry loop. +/// +/// +/// Every log line carries the envelope's own correlation_id, read as a raw string. +/// Three of the four commands also carry a domain correlation, but AlarmCommand has no +/// CLR member for one, so the envelope field is the only id every command shares — and it is +/// still readable when the inner fields are what failed to parse, which is precisely when an +/// operator needs it. +/// +/// +/// Inbound auth is MeshCommandAuthInterceptor, scoped to this service's method prefix and +/// registered at the AddGrpc level; it fail-closes with no MeshTransport:GrpcPsk +/// configured. Mapping this service is what makes that gate live. +/// +/// +public sealed class NodeCommandGrpcService : GeneratedServiceBase +{ + private readonly ActorRegistry _registry; + private readonly ILogger _log; + + /// Initializes a new instance of the class. + /// Actor registry holding this node's NodeCommunicationActor. + /// Logger. + public NodeCommandGrpcService(ActorRegistry registry, ILogger log) + { + _registry = registry; + _log = log; + } + + /// + public override Task Execute(CommandEnvelope request, ServerCallContext context) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(context); + + object command; + try + { + command = MeshTransportDtoMapper.FromEnvelope(request); + } + catch (Exception ex) when (ex is InvalidOperationException or FormatException or ArgumentException) + { + // The envelope carries no credential material, so logging its correlation id is safe and is + // the only handle an operator has on a call whose inner fields did not parse. + _log.LogWarning( + ex, + "Rejected a mesh command (correlationId={CorrelationId}, oneof={CommandCase}): the " + + "envelope could not be decoded. The command was NOT delivered.", + request.CorrelationId, request.CommandCase); + throw new RpcException(new Status( + StatusCode.InvalidArgument, "Command envelope could not be decoded.")); + } + + if (!_registry.TryGet(out var comm)) + { + _log.LogWarning( + "Rejected a mesh command (correlationId={CorrelationId}, type={CommandType}): this node's " + + "communication actor is not registered yet. The node is still starting; the command was " + + "NOT delivered and central may retry.", + request.CorrelationId, command.GetType().Name); + throw new RpcException(new Status( + StatusCode.Unavailable, "Node is still starting: the command endpoint is not ready.")); + } + + comm.Tell(command); + + _log.LogDebug( + "Mesh: accepted {CommandType} (correlationId={CorrelationId}) over gRPC; handed to the node " + + "communication actor.", + command.GetType().Name, request.CorrelationId); + + return Task.FromResult(new ExecuteAck()); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index 8fb24e89..33ff5a43 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -740,9 +740,18 @@ if (hasAdmin && configServeGrpcPort > 0) // Driver-node live-telemetry stream (per-cluster mesh Phase 5). Central dials in; gated by // TelemetryStreamAuthInterceptor (fail-closed on empty Telemetry:ApiKey). +// +// The node-side command endpoint (ClusterClient→gRPC migration Phase 1) rides the SAME driver-role +// h2c listener rather than claiming a fourth port: central already dials this node here, both +// directions are per-cluster-keyed, and a separate port would be one more thing to open on every +// site firewall for no isolation gain. Mapping it is what makes MeshCommandAuthInterceptor live — +// it is registered above and is a pass-through until a call matches this service's method prefix. +// It fail-closes with no MeshTransport:GrpcPsk, so an operator who maps the service without setting +// a key gets a closed endpoint, not an open one. if (hasDriver && telemetryListenPort > 0) { app.MapGrpcService(); + app.MapGrpcService(); } app.MapOtOpcUaHealth(); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/AssemblyInfo.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/AssemblyInfo.cs index 63dc64f4..26a08294 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/AssemblyInfo.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/AssemblyInfo.cs @@ -2,5 +2,6 @@ using Xunit; // The TelemetryStreamGrpcService concurrency counter is process-wide (static). Run this assembly's // tests serially so the cap test observes a deterministic active-stream count with no cross-test -// interference. This project holds only telemetry tests, so serial execution costs nothing. +// interference. Every other suite here is small and fast (the mesh-command tests spin a TestKit +// ActorSystem per class), so serial execution costs nothing. [assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Grpc/NodeCommandGrpcServiceTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Grpc/NodeCommandGrpcServiceTests.cs new file mode 100644 index 00000000..0ccd6ea8 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Grpc/NodeCommandGrpcServiceTests.cs @@ -0,0 +1,186 @@ +using Akka.Hosting; +using Akka.TestKit.Xunit2; +using Grpc.Core; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Host.Grpc; +using ZB.MOM.WW.OtOpcUa.Runtime; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Grpc; + +/// +/// Verifies the node-side command ingress (ClusterClient→gRPC migration Phase 1): each of the four +/// central→node commands is decoded off its and handed to the +/// registered NodeCommunicationActor as the same CLR record the ClusterClient transport +/// delivered, an undecodable envelope is refused with , and +/// a call landing in the boot window before the actor is registered is refused with +/// . +/// +/// +/// The actor is a TestProbe registered under : what +/// this service owes the rest of the system is the exact message the real actor already knows how to +/// re-emit, so asserting on what arrives at the registry key is the whole contract. The onward leg — +/// comm actor to EventStream to the real driver host — is pinned unchanged by +/// EventStreamCommandDeliveryTests in the Runtime test project. +/// +public sealed class NodeCommandGrpcServiceTests : TestKit +{ + private static readonly CorrelationId AlarmCorrelation = new(Guid.NewGuid()); + + private static ServerCallContext Ctx() => new FakeServerCallContext(); + + private static NodeCommandGrpcService NewService(ActorRegistry registry) => + new(registry, NullLogger.Instance); + + [Fact] + public async Task Each_of_the_four_commands_reaches_the_comm_actor_as_its_domain_record() + { + var probe = CreateTestProbe(); + var registry = new ActorRegistry(); + registry.Register(probe.Ref); + var service = NewService(registry); + + var dispatch = new DispatchDeployment( + new DeploymentId(Guid.NewGuid()), new RevisionHash("rev-1"), new CorrelationId(Guid.NewGuid())); + var restart = new RestartDriver("cluster-a", "drv-1", "alice", Guid.NewGuid()); + var reconnect = new ReconnectDriver("cluster-a", "drv-2", "bob", Guid.NewGuid()); + var alarm = new AlarmCommand("alarm-7", "Acknowledge", "carol", "looked at it", null); + + (await service.Execute(MeshTransportDtoMapper.ToEnvelope(dispatch), Ctx())).ShouldNotBeNull(); + probe.ExpectMsg().ShouldBe(dispatch); + + await service.Execute(MeshTransportDtoMapper.ToEnvelope(restart), Ctx()); + probe.ExpectMsg().ShouldBe(restart); + + await service.Execute(MeshTransportDtoMapper.ToEnvelope(reconnect), Ctx()); + probe.ExpectMsg().ShouldBe(reconnect); + + await service.Execute(MeshTransportDtoMapper.ToEnvelope(alarm, AlarmCorrelation), Ctx()); + probe.ExpectMsg().ShouldBe(alarm); + } + + [Fact] + public async Task A_call_in_the_boot_window_before_the_actor_is_registered_is_unavailable() + { + // Nothing registered: the ActorSystem exists but AddOtOpcUaRuntime has not spawned the comm + // actor yet. Unavailable (not Internal) because the condition is transient and retryable — + // central's dialer may try the same node again a moment later and succeed. + var service = NewService(new ActorRegistry()); + var dispatch = new DispatchDeployment( + new DeploymentId(Guid.NewGuid()), new RevisionHash("rev-1"), new CorrelationId(Guid.NewGuid())); + + var ex = await Should.ThrowAsync( + () => service.Execute(MeshTransportDtoMapper.ToEnvelope(dispatch), Ctx())); + + ex.StatusCode.ShouldBe(StatusCode.Unavailable); + } + + [Fact] + public async Task An_unset_oneof_is_invalid_argument() + { + var probe = CreateTestProbe(); + var registry = new ActorRegistry(); + registry.Register(probe.Ref); + var service = NewService(registry); + + var ex = await Should.ThrowAsync(() => service.Execute( + new CommandEnvelope { CorrelationId = Guid.NewGuid().ToString("N") }, Ctx())); + + ex.StatusCode.ShouldBe(StatusCode.InvalidArgument); + probe.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); + } + + [Theory] + [MemberData(nameof(UndecodableEnvelopes))] + public async Task An_envelope_whose_fields_do_not_parse_is_invalid_argument(CommandEnvelope envelope) + { + var probe = CreateTestProbe(); + var registry = new ActorRegistry(); + registry.Register(probe.Ref); + var service = NewService(registry); + + var ex = await Should.ThrowAsync(() => service.Execute(envelope, Ctx())); + + ex.StatusCode.ShouldBe(StatusCode.InvalidArgument); + probe.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); + } + + public static TheoryData UndecodableEnvelopes() => + [ + // Deployment id is not a hyphenless GUID -> FormatException out of the mapper. + new CommandEnvelope + { + CorrelationId = Guid.NewGuid().ToString("N"), + DispatchDeployment = new DispatchDeploymentCommand + { + DeploymentId = "not-a-guid", + RevisionHash = "rev-1", + CorrelationId = Guid.NewGuid().ToString("N"), + }, + }, + // Revision hash is blank -> ArgumentException out of RevisionHash.Parse. + new CommandEnvelope + { + CorrelationId = Guid.NewGuid().ToString("N"), + DispatchDeployment = new DispatchDeploymentCommand + { + DeploymentId = Guid.NewGuid().ToString("N"), + RevisionHash = "", + CorrelationId = Guid.NewGuid().ToString("N"), + }, + }, + // Restart command with an unparseable correlation id -> FormatException. + new CommandEnvelope + { + CorrelationId = Guid.NewGuid().ToString("N"), + RestartDriver = new RestartDriverCommand + { + ClusterId = "cluster-a", + DriverInstanceId = "drv-1", + ActorByUserName = "alice", + CorrelationId = "nope", + }, + }, + ]; + + [Fact] + public async Task An_undecodable_envelope_in_the_boot_window_is_invalid_argument_not_unavailable() + { + // Ordering pin: decode runs BEFORE the registry lookup. Unavailable invites the caller to + // retry, and an envelope this build cannot decode will never decode — reporting it as a + // readiness problem would turn a permanent fault into an unbounded retry loop. + var service = NewService(new ActorRegistry()); + + var ex = await Should.ThrowAsync(() => service.Execute( + new CommandEnvelope { CorrelationId = Guid.NewGuid().ToString("N") }, Ctx())); + + ex.StatusCode.ShouldBe(StatusCode.InvalidArgument); + } + + /// Minimal ; this RPC reads nothing off the context but its method name. + private sealed class FakeServerCallContext : ServerCallContext + { + protected override string MethodCore => "/mesh_transport.v1.NodeCommandService/Execute"; + protected override string HostCore => "localhost"; + protected override string PeerCore => "ipv4:127.0.0.1:0"; + protected override DateTime DeadlineCore => DateTime.MaxValue; + protected override Metadata RequestHeadersCore => []; + protected override CancellationToken CancellationTokenCore => CancellationToken.None; + protected override Metadata ResponseTrailersCore { get; } = []; + protected override Status StatusCore { get; set; } + protected override WriteOptions? WriteOptionsCore { get; set; } + protected override AuthContext AuthContextCore => new(null, new Dictionary>()); + + protected override ContextPropagationToken CreatePropagationTokenCore(ContextPropagationOptions? options) => + throw new NotSupportedException(); + + protected override Task WriteResponseHeadersAsyncCore(Metadata responseHeaders) => Task.CompletedTask; + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/ZB.MOM.WW.OtOpcUa.Host.Tests.csproj b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/ZB.MOM.WW.OtOpcUa.Host.Tests.csproj index 7798ef15..b4d6f70e 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/ZB.MOM.WW.OtOpcUa.Host.Tests.csproj +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/ZB.MOM.WW.OtOpcUa.Host.Tests.csproj @@ -10,6 +10,9 @@ + + From 2edef72f6fc39d1b27e74f0ac4a3de64e81ade3e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 6 Aug 2026 05:19:07 -0400 Subject: [PATCH 02/12] =?UTF-8?q?docs(plans):=201A.1=20close-out=20?= =?UTF-8?q?=E2=80=94=20reviews=20=E2=9C=85,=20plan=20FromEnvelope=20wordin?= =?UTF-8?q?g=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plans/2026-08-06-clusterclient-to-grpc.md | 2 +- docs/plans/2026-08-06-clusterclient-to-grpc.md.tasks.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/plans/2026-08-06-clusterclient-to-grpc.md b/docs/plans/2026-08-06-clusterclient-to-grpc.md index a92464a1..bdb0433f 100644 --- a/docs/plans/2026-08-06-clusterclient-to-grpc.md +++ b/docs/plans/2026-08-06-clusterclient-to-grpc.md @@ -248,7 +248,7 @@ user-owned.** - Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Grpc/NodeCommandGrpcServiceTests.cs` Follow the alias convention (`using GeneratedServiceBase = …NodeCommandService.NodeCommandServiceBase;`, -class suffixed `GrpcService`). `Execute`: decode via `MeshTransportDtoMapper.FromProto`, Tell +class suffixed `GrpcService`). `Execute`: decode via `MeshTransportDtoMapper.FromEnvelope`, Tell the registered `NodeCommunicationActor` (resolve via `NodeCommunicationActorKey` registry, `Runtime/ServiceCollectionExtensions.cs:583`), return `ExecuteAck`. Actor not yet registered (boot window) ⇒ `RpcException(Unavailable)` — the readiness convention. Decode failure ⇒ diff --git a/docs/plans/2026-08-06-clusterclient-to-grpc.md.tasks.json b/docs/plans/2026-08-06-clusterclient-to-grpc.md.tasks.json index 2ad26b7f..86202070 100644 --- a/docs/plans/2026-08-06-clusterclient-to-grpc.md.tasks.json +++ b/docs/plans/2026-08-06-clusterclient-to-grpc.md.tasks.json @@ -9,8 +9,8 @@ {"id": 3, "subject": "Task 0.4: node-side MeshCommandAuthInterceptor", "phase": "0", "branch": "feat/grpc-phase0-contract-auth", "classification": "high-risk", "parallelizableWith": ["0.5 (batch, same agent)"], "status": "completed", "blockedBy": [2], "commit": "fbda015c + fixes 24e55d33/a4631410", "review": "spec ✅ + code ✅ (opus, fix loop closed)"}, {"id": 4, "subject": "Task 0.5: IMeshPskProvider + CentralCommandAuthInterceptor", "phase": "0", "branch": "feat/grpc-phase0-contract-auth", "classification": "high-risk", "parallelizableWith": ["0.4 (batch, same agent)"], "status": "completed", "blockedBy": [2], "commit": "68a416e4 + fixes 24e55d33/a4631410", "review": "spec ✅ + code ✅ (opus, fix loop closed); 4 items gated on Phase 1A — see tracking §Phase 1A security register"}, {"id": 5, "subject": "Task 0.6: rig config + FleetStatus issue + docs stub + Phase 0 gate", "phase": "0", "branch": "feat/grpc-phase0-contract-auth", "classification": "small", "parallelizableWith": [], "status": "completed", "blockedBy": [0, 1, 2, 3, 4], "commit": "23555ed0", "review": "code ✅", "gate": "PASS 4/4; issues #542 #543 filed"}, - {"id": 6, "subject": "Task 1A.1: NodeCommandGrpcService on the telemetry listener", "phase": "1A", "branch": "feat/grpc-node-command", "classification": "standard", "parallelizableWith": ["1B.1 (other worktree)"], "status": "pending", "blockedBy": [5]}, - {"id": 7, "subject": "Task 1A.2: MeshGrpcCommandDialer + third branch in CentralCommunicationActor", "phase": "1A", "branch": "feat/grpc-node-command", "classification": "high-risk", "parallelizableWith": ["1B.2 (other worktree)"], "status": "pending", "blockedBy": [6]}, + {"id": 6, "subject": "Task 1A.1: NodeCommandGrpcService on the telemetry listener", "phase": "1A", "branch": "feat/grpc-node-command", "classification": "standard", "parallelizableWith": ["1B.1 (other worktree)"], "status": "completed", "blockedBy": [5], "commit": "64774652", "review": "spec ✅ + code ✅"}, + {"id": 7, "subject": "Task 1A.2: MeshGrpcCommandDialer + third branch in CentralCommunicationActor (+ security register items 1-2)", "phase": "1A", "branch": "feat/grpc-node-command", "classification": "high-risk", "parallelizableWith": ["1B.2 (other worktree)"], "status": "in_progress", "blockedBy": [6]}, {"id": 8, "subject": "Task 1A.3: 1A live gate (central pair flipped, rig-only, reverted)", "phase": "1A", "branch": "feat/grpc-node-command", "classification": "small", "parallelizableWith": [], "status": "pending", "blockedBy": [7]}, {"id": 9, "subject": "Task 1B.1: CentralCommandGrpcService + admin validator rule", "phase": "1B", "branch": "feat/grpc-central-ack", "classification": "standard", "parallelizableWith": ["1A.1 (other worktree)"], "status": "pending", "blockedBy": [5]}, {"id": 10, "subject": "Task 1B.2: GrpcApplyAckSender + node-side wiring", "phase": "1B", "branch": "feat/grpc-central-ack", "classification": "high-risk", "parallelizableWith": ["1A.2 (other worktree)"], "status": "pending", "blockedBy": [9]}, From 14d21496fee4405c404e48647b8c0b05e044f01a Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 6 Aug 2026 05:31:08 -0400 Subject: [PATCH 03/12] feat(mesh): known-cluster gate + Matches on the PSK provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1A security register items 1 and 2, landed ahead of the command fan-out because they are independently testable and the fan-out is the first caller of TryGetPsk. Item 1 — known-cluster gate. MeshPskProvider gains SetKnownClusters, an atomic whole-set swap consulted by TryGetPsk BEFORE the config map and before the secret store. A cluster id outside the set is refused in O(1) with no store read and no audit write, which closes two things at once: the pre-auth store/audit flood an unauthenticated caller could drive with a forged x-otopcua-cluster header, and the timing oracle that defeated CentralCommandAuthInterceptor's uniform denial text (unknown clusters are never cached, so they paid a store read on every probe while known ones answered from memory). Empty or never-published leaves the gate inactive — central publishes the set from its DB contact refresh, and 'the DB told me nothing' must not become a fleet-wide authentication outage during admin boot. Item 2 — bool Matches(clusterId, presented) on IMeshPskProvider, with FixedTimeEquals inside the provider. CentralCommandAuthInterceptor now calls it and never holds key material; its two provider-side denial logs collapse into one (the provider answers both as false) and the operator hint is restated there. No production caller publishes a cluster set yet, so behaviour is unchanged until the fan-out commit wires the contact refresh. ControlPlane.Tests 190 -> 205, Host.Tests 91 -> 92. --- .../Communication/IMeshPskProvider.cs | 118 ++++++++++- .../CentralCommandAuthInterceptor.cs | 67 +++--- .../Communication/MeshPskProviderTests.cs | 199 ++++++++++++++++++ .../CentralCommandAuthInterceptorTests.cs | 44 +++- 4 files changed, 392 insertions(+), 36 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs index 61d37f74..a7282694 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs @@ -1,5 +1,7 @@ using System.Collections.Concurrent; using System.Diagnostics.CodeAnalysis; +using System.Security.Cryptography; +using System.Text; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ZB.MOM.WW.OtOpcUa.Cluster; @@ -21,10 +23,21 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; /// proceed unauthenticated; a missing key must cost delivery, not authentication. /// /// -/// The Invalidate hooks exist because central's cluster set is DB-dynamic. They are -/// deliberately unconsumed at Phase 0 — Phase 1A wires them to the contact refresh, which is -/// the point at which central learns a cluster's membership (and so possibly its key) has -/// changed. +/// The Invalidate hooks exist because central's cluster set is DB-dynamic. Phase 1A +/// wires them to CentralCommunicationActor's contact refresh, which is the point at +/// which central learns a cluster's membership (and so possibly its key) has changed — so a +/// rotated key is picked up within one ContactRefreshSeconds without a restart. +/// +/// +/// The known-cluster gate is a real security control, not an optimization (Phase 1A +/// security register item 1). publishes the cluster ids +/// central just read from ClusterNode; consults that set +/// first and refuses anything outside it in O(1), touching neither the config map nor the +/// secret store. Without it, an unauthenticated caller can pick an arbitrary +/// x-otopcua-cluster value and make central perform a secret-store read plus an audit +/// write per probe — and, because negatives are deliberately never cached, the cost +/// difference between a known and an unknown cluster is measurable from the wire, which +/// defeats the uniform denial text on the central interceptor. /// /// /// Invalidate is not a revocation barrier. It drops the cached entry; it does @@ -43,6 +56,50 @@ public interface IMeshPskProvider /// true when a usable key was found; otherwise false. bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk); + /// + /// Publishes the set of application Cluster ids central currently knows about. + /// refuses anything outside it before consulting any source. + /// + /// + /// The known cluster ids, matched case-insensitively. An empty collection deactivates the + /// gate rather than refusing everything: central publishes this from its DB contact + /// refresh, and "the DB told me nothing" must not become a fleet-wide authentication outage. + /// Never having called this leaves the gate inactive for the same reason — the central + /// interceptor is live from the first request, before the first refresh has run. + /// + /// + /// Implementations must take a copy: the caller owns its collection and may reuse it. The + /// swap must also be atomic with respect to concurrent callers, + /// which run on arbitrary gRPC threads. + /// + void SetKnownClusters(IReadOnlyCollection clusterIds); + + /// + /// Verifies a presented bearer token against one cluster's key, in constant time with + /// respect to the key's content. + /// + /// The application Cluster the caller claims to be. + /// The bearer token the caller supplied. + /// true only when a key exists for that cluster and the token equals it. + /// + /// + /// This is the only comparison on the verify path (Phase 1A security register + /// item 2). Handing the raw key back so a caller can compare it is how == + /// eventually appears in an interceptor and compiles; keeping the comparison behind this + /// member means the verify side never holds key material at all. + /// survives for the dial side, which has to send the key rather than compare it. + /// + /// + /// Declared without a default implementation on purpose: a security member a new + /// implementer can inherit silently is a member a new implementer never considers. + /// + /// + /// Length differences remain observable through timing; that is inherent to comparing + /// byte arrays of different lengths and is not sensitive. Content is not. + /// + /// + bool Matches(string clusterId, string presented); + /// Drops any cached key for one cluster, so the next lookup re-reads the store. /// The application Cluster id, matched case-insensitively. void Invalidate(string clusterId); @@ -87,6 +144,18 @@ public sealed class MeshPskProvider : IMeshPskProvider private readonly ConcurrentDictionary> _cache = new(StringComparer.OrdinalIgnoreCase); + /// + /// The known-cluster gate's membership set, or when the gate is + /// inactive (never published, or published empty — see ). + /// + /// + /// Swapped wholesale, never mutated in place, so a reader either sees the previous set or the + /// next one and never a half-built one. volatile is what makes the publish visible to + /// the gRPC threads doing the reading; no lock is needed because there is exactly one writer + /// (the contact-refresh handler, on the actor's own thread). + /// + private volatile IReadOnlySet? _knownClusters; + /// Creates the provider. /// Mesh transport options; ClusterPsks is the override map. /// The shared secret store read seam. @@ -118,6 +187,19 @@ public sealed class MeshPskProvider : IMeshPskProvider return false; } + // The known-cluster gate, ahead of BOTH sources. Deliberately silent: the caller that + // rejected the call already logs the denial, and a second line here would be a log flood + // driven by exactly the unauthenticated traffic this gate exists to make cheap. + // + // Ahead of the config map too, not just the store. That has a visible consequence — a + // MeshTransport:ClusterPsks entry for a cluster with no enabled, non-maintenance ClusterNode + // row stops resolving once central publishes its set — and it is the right one: central + // would neither dial such a cluster nor accept an ack from it, so serving a key for it only + // widens what an unauthenticated caller can probe. + var known = _knownClusters; + if (known is not null && !known.Contains(clusterId)) + return false; + // Config map first, on every call. Blank counts as absent: an empty // MeshTransport__ClusterPsks__ is an unset environment variable, not an operator // deciding this cluster has an empty key. @@ -168,6 +250,34 @@ public sealed class MeshPskProvider : IMeshPskProvider return true; } + /// + public bool Matches(string clusterId, string presented) + { + // Blank is refused on its own terms rather than by relying on the fact that a blank key + // never survives resolution: this is an authentication gate, and "both sides happened to be + // empty" must never be a pass. + if (string.IsNullOrWhiteSpace(presented)) + return false; + + if (!TryGetPsk(clusterId, out var expected)) + return false; + + return CryptographicOperations.FixedTimeEquals( + Encoding.UTF8.GetBytes(presented), Encoding.UTF8.GetBytes(expected)); + } + + /// + public void SetKnownClusters(IReadOnlyCollection clusterIds) + { + ArgumentNullException.ThrowIfNull(clusterIds); + + // Copied, so a caller reusing its list cannot widen the gate after the fact; empty collapses + // to null so the gate has exactly one "inactive" representation to test for. + _knownClusters = clusterIds.Count == 0 + ? null + : clusterIds.ToHashSet(StringComparer.OrdinalIgnoreCase); + } + /// public void Invalidate(string clusterId) { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/CentralCommandAuthInterceptor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/CentralCommandAuthInterceptor.cs index 3cedf15f..3b1f635a 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/CentralCommandAuthInterceptor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/CentralCommandAuthInterceptor.cs @@ -1,5 +1,3 @@ -using System.Security.Cryptography; -using System.Text; using Grpc.Core; using Grpc.Core.Interceptors; using Microsoft.Extensions.Logging; @@ -38,25 +36,31 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Configuration; /// key this design exists to avoid. /// /// -/// Comparison is over UTF-8 bytes, so a -/// wrong key cannot be recovered byte-by-byte from response timing. Length differences are -/// unavoidably observable and are not sensitive. +/// This interceptor never handles a key. Verification is +/// , which owns the constant-time comparison, so a +/// wrong key cannot be recovered byte-by-byte from response timing and no future edit here +/// can replace that comparison with == and still compile (Phase 1A security register +/// item 2). Length differences are unavoidably observable and are not sensitive. /// /// -/// All three denials return the same status text, unlike the sibling interceptors, -/// which distinguish "no key configured" from "authentication failed". Here that distinction -/// would be a content oracle: the caller supplies the cluster id, so a reply that said -/// "no key for that cluster" would let an unauthenticated caller read central's cluster set -/// straight off the wire. The operator-facing detail goes to the log instead. +/// All denials return the same status text, unlike the sibling interceptors, which +/// distinguish "no key configured" from "authentication failed". Here that distinction would +/// be a content oracle: the caller supplies the cluster id, so a reply that said "no +/// key for that cluster" would let an unauthenticated caller read central's cluster set +/// straight off the wire. The operator-facing detail goes to the log instead — and for the +/// same reason the log no longer separates "no key for that cluster" from "wrong key": the +/// provider answers both as one false, and collapsing them costs an operator a hint +/// (restated in the one denial message) rather than costing a diagnosis. /// /// -/// That is not yet full enumeration resistance, and should not be read as such. A -/// timing oracle survives it: an unknown cluster misses the provider's cache and pays a store -/// read on every probe, while a known one is answered from memory after the first. Closing -/// that needs the lookup to be refused before it reaches the store for any cluster central -/// does not know — which is the Phase 1A known-cluster-set gate, where central learns its -/// cluster set from the DB. Until then, uniform text buys content-level indistinguishability -/// only. +/// The timing oracle that survived Phase 0 is closed by the known-cluster gate. An +/// unknown cluster used to miss the provider's cache and pay a secret-store read on every +/// probe, while a known one was answered from memory after the first — a stable, remotely +/// measurable distinguisher, and a pre-auth store/audit flood besides. +/// CentralCommunicationActor now publishes central's cluster set to the provider on +/// every contact refresh, and a cluster outside it is refused in O(1) with no store touch, +/// so the two cases cost the same. What remains is a known cluster's first-ever lookup, +/// which is one read per cluster per process and not attacker-triggerable. /// /// /// Exactly one public constructor. Grpc.AspNetCore silently stops invoking an @@ -151,13 +155,15 @@ public sealed class CentralCommandAuthInterceptor : Interceptor "Central command authentication failed.")); } - if (!_pskProvider.TryGetPsk(clusterId, out var expected)) + // Checked here rather than deferred to Matches so the log can still separate "the caller + // presented nothing" — an operator misconfiguration, and the one case the provider cannot + // see — from "the caller presented something that did not verify". + var presented = GrpcRequestHeaders.BearerToken(context.RequestHeaders); + if (presented is null) { _logger.LogWarning( - "Rejected a central command call to {Method} claiming cluster {ClusterId}: no " - + "pre-shared key could be established for it. Seed the secret " - + "OT-MESH-PSK- on this node, or set " - + "MeshTransport:ClusterPsks: on a rig.", + "Rejected a central command call to {Method} claiming cluster {ClusterId}: no bearer " + + "token presented.", context.Method, clusterId); throw new RpcException(new Status( @@ -165,21 +171,20 @@ public sealed class CentralCommandAuthInterceptor : Interceptor "Central command authentication failed.")); } - var presented = GrpcRequestHeaders.BearerToken(context.RequestHeaders); - if (presented is null || !FixedTimeEquals(presented, expected)) + if (!_pskProvider.Matches(clusterId, presented)) { _logger.LogWarning( - "Rejected a central command call to {Method} claiming cluster {ClusterId}: {Reason}.", + "Rejected a central command call to {Method} claiming cluster {ClusterId}: the " + + "presented bearer token did not match that cluster's pre-shared key, no key could " + + "be established for it, or it is not a cluster this node knows. If the cluster is " + + "legitimate, seed the secret OT-MESH-PSK- on this node, or set " + + "MeshTransport:ClusterPsks: on a rig, and check it has an enabled, " + + "non-maintenance ClusterNode row.", context.Method, - clusterId, - presented is null ? "no bearer token presented" : "bearer token did not match"); + clusterId); throw new RpcException(new Status( StatusCode.PermissionDenied, "Central command authentication failed.")); } } - - private static bool FixedTimeEquals(string presented, string expected) - => CryptographicOperations.FixedTimeEquals( - Encoding.UTF8.GetBytes(presented), Encoding.UTF8.GetBytes(expected)); } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderTests.cs index b87daa32..3360ab9f 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderTests.cs @@ -367,6 +367,205 @@ public sealed class MeshPskProviderTests resolver.CallCount.ShouldBe(0); } + // ------------------------------------------------------------------ known-cluster gate + // + // Phase 1A security register item 1. Central rebuilds its cluster set from ClusterNode rows on + // every contact refresh; a cluster id outside that set must be refused in O(1) BEFORE anything + // touches the secret store. Two things ride on that: an unauthenticated caller can no longer + // make central perform a store read (and an audit write) per forged x-otopcua-cluster header, + // and the timing difference between a known cluster (cached) and an unknown one (never cached, + // because negatives are deliberately not retained) stops being remotely measurable. + + [Fact] + public void UnknownCluster_WithTheGatePopulated_IsRefused_WithoutTouchingTheStore() + { + // THE assertion of the register item. CallCount == 0 is the whole point: the refusal must + // cost nothing, or the gate is a comment rather than a control. + var resolver = new FakeSecretResolver { Value = "stored-psk" }; + var provider = CreateProvider(resolver); + provider.SetKnownClusters(["SITE-A", "SITE-B"]); + + provider.TryGetPsk("SITE-Z", out var psk).ShouldBeFalse(); + + psk.ShouldBeNull(); + resolver.CallCount.ShouldBe(0); + } + + [Fact] + public void KnownCluster_WithTheGatePopulated_StillResolves() + { + var resolver = new FakeSecretResolver { Value = "stored-psk" }; + var provider = CreateProvider(resolver); + provider.SetKnownClusters(["SITE-A"]); + + provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); + + psk.ShouldBe("stored-psk"); + resolver.CallCount.ShouldBe(1); + } + + [Fact] + public void TheKnownClusterSet_IsCaseInsensitive() + { + // ClusterId case is whatever the operator typed into the DB row; the header is whatever the + // node was configured with. A case-sensitive gate would refuse a legitimate cluster. + var resolver = new FakeSecretResolver { Value = "stored-psk" }; + var provider = CreateProvider(resolver); + provider.SetKnownClusters(["SITE-A"]); + + provider.TryGetPsk("site-a", out var psk).ShouldBeTrue(); + + psk.ShouldBe("stored-psk"); + } + + [Fact] + public void SwappingTheKnownClusterSet_AddsAndRemovesClusters() + { + // The set is rebuilt from the DB every refresh, so both directions happen in production: a + // new site appears, a retired one stops being accepted. + var resolver = new FakeSecretResolver { Value = "stored-psk" }; + var provider = CreateProvider(resolver); + + provider.SetKnownClusters(["SITE-A"]); + provider.TryGetPsk("SITE-A", out _).ShouldBeTrue(); + provider.TryGetPsk("SITE-B", out _).ShouldBeFalse(); + + provider.SetKnownClusters(["SITE-B"]); + provider.TryGetPsk("SITE-B", out _).ShouldBeTrue(); + provider.TryGetPsk("SITE-A", out _).ShouldBeFalse(); + } + + [Fact] + public void AnEmptyKnownClusterSet_LeavesTheGateInactive() + { + // Fail-OPEN, deliberately and narrowly: central publishes the set from its contact refresh, + // and an empty publish means "the DB told me nothing", not "no cluster is legitimate". + // Refusing everything there would make a fleet-wide ack outage the failure mode of an empty + // or unreadable ClusterNode table. + var resolver = new FakeSecretResolver { Value = "stored-psk" }; + var provider = CreateProvider(resolver); + provider.SetKnownClusters(["SITE-A"]); + provider.SetKnownClusters([]); + + provider.TryGetPsk("SITE-Z", out var psk).ShouldBeTrue(); + + psk.ShouldBe("stored-psk"); + } + + [Fact] + public void AnUnsetKnownClusterSet_LeavesTheGateInactive() + { + // Admin boot: the interceptor is live from the first request, but the first contact refresh + // has not run yet. Gating on a set nobody has published would deny every ack in that window. + var resolver = new FakeSecretResolver { Value = "stored-psk" }; + var provider = CreateProvider(resolver); + + provider.TryGetPsk("SITE-Z", out var psk).ShouldBeTrue(); + + psk.ShouldBe("stored-psk"); + } + + [Fact] + public void TheGate_IsConsultedBeforeTheConfigMap() + { + // Consequence worth pinning rather than discovering: a ClusterPsks entry for a cluster with + // no enabled ClusterNode row is refused once the gate is populated. That is correct — central + // would neither dial such a cluster nor accept an ack from it — but it is surprising enough + // on a rig to deserve a test that names it. + var resolver = new FakeSecretResolver(); + var provider = CreateProvider( + resolver, new Dictionary { ["SITE-X"] = "rig-psk-site-x" }); + provider.SetKnownClusters(["SITE-A"]); + + provider.TryGetPsk("SITE-X", out var psk).ShouldBeFalse(); + + psk.ShouldBeNull(); + } + + [Fact] + public void SetKnownClusters_TakesACopy_SoALaterMutationOfTheCallersList_CannotWidenTheGate() + { + var clusters = new List { "SITE-A" }; + var resolver = new FakeSecretResolver { Value = "stored-psk" }; + var provider = CreateProvider(resolver); + provider.SetKnownClusters(clusters); + + clusters.Add("SITE-Z"); + + provider.TryGetPsk("SITE-Z", out _).ShouldBeFalse(); + } + + // ------------------------------------------------------------------ Matches + + [Fact] + public void Matches_AcceptsTheClustersOwnKey() + { + var resolver = new FakeSecretResolver(); + var provider = CreateProvider( + resolver, new Dictionary { ["SITE-A"] = "psk-site-a" }); + + provider.Matches("SITE-A", "psk-site-a").ShouldBeTrue(); + } + + [Fact] + public void Matches_RejectsAnotherClustersKey() + { + // The isolation property, asserted at the provider rather than only at the interceptor: + // choosing a cluster only changes which key you are graded against. + var resolver = new FakeSecretResolver(); + var provider = CreateProvider( + resolver, + new Dictionary { ["SITE-A"] = "psk-site-a", ["SITE-B"] = "psk-site-b" }); + + provider.Matches("SITE-A", "psk-site-b").ShouldBeFalse(); + } + + [Fact] + public void Matches_IsNotAPrefixMatch() + { + var resolver = new FakeSecretResolver(); + var provider = CreateProvider( + resolver, new Dictionary { ["SITE-A"] = "psk-site-a" }); + + provider.Matches("SITE-A", "psk-site-").ShouldBeFalse(); + provider.Matches("SITE-A", "psk-site-a-and-more").ShouldBeFalse(); + } + + [Fact] + public void Matches_FailsClosed_WhenTheClusterHasNoKey() + { + var resolver = new FakeSecretResolver { Value = null }; + var provider = CreateProvider(resolver); + + provider.Matches("SITE-Z", "anything-at-all").ShouldBeFalse(); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + public void Matches_RejectsABlankPresentedToken(string presented) + { + // A blank key never reaches the cache (blank resolves fail closed), but a blank PRESENTED + // token must be refused on its own terms rather than relying on that. + var resolver = new FakeSecretResolver(); + var provider = CreateProvider( + resolver, new Dictionary { ["SITE-A"] = "psk-site-a" }); + + provider.Matches("SITE-A", presented).ShouldBeFalse(); + } + + [Fact] + public void Matches_IsRefusedByTheKnownClusterGate_WithoutTouchingTheStore() + { + var resolver = new FakeSecretResolver { Value = "stored-psk" }; + var provider = CreateProvider(resolver); + provider.SetKnownClusters(["SITE-A"]); + + provider.Matches("SITE-Z", "stored-psk").ShouldBeFalse(); + + resolver.CallCount.ShouldBe(0); + } + /// /// An whose answer the test controls between calls, so cache /// staleness and invalidation are observable without a real store. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/CentralCommandAuthInterceptorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/CentralCommandAuthInterceptorTests.cs index 026d9030..bb619709 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/CentralCommandAuthInterceptorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/CentralCommandAuthInterceptorTests.cs @@ -242,14 +242,56 @@ public sealed class CentralCommandAuthInterceptorTests publicCtors.Length.ShouldBe(1); } + [Fact] + public async Task Verification_GoesThroughMatches_AndTheInterceptorNeverHandlesTheKeyItself() + { + // Phase 1A security register item 2. The interceptor must not fetch the raw key and compare + // it — that is how a constant-time comparison eventually becomes '=='. Matches owns the + // comparison; TryGetPsk survives only for the DIAL side, which has to send the key. + // (The provider's own default Matches calls TryGetPsk internally; this fake overrides it, so + // a zero TryGetPsk count here means the INTERCEPTOR never asked for the key.) + var provider = new FakeMeshPskProvider(("SITE-A", "psk-site-a")); + var interceptor = new CentralCommandAuthInterceptor( + provider, NullLogger.Instance); + + await Invoke(interceptor, CreateContext(ReportApplyMethod, "Bearer psk-site-a", "SITE-A")); + + provider.MatchesCalls.ShouldBe(1); + provider.TryGetPskCalls.ShouldBe(0); + } + /// An over a fixed key set; unknown clusters fail closed. private sealed class FakeMeshPskProvider(params (string Cluster, string Psk)[] keys) : IMeshPskProvider { private readonly Dictionary _keys = keys.ToDictionary(k => k.Cluster, k => k.Psk, StringComparer.OrdinalIgnoreCase); + /// How many times the raw key was asked for — must stay 0 on the verify path. + public int TryGetPskCalls { get; private set; } + + /// How many times verification went through the provider's comparison. + public int MatchesCalls { get; private set; } + public bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk) - => _keys.TryGetValue(clusterId, out psk); + { + TryGetPskCalls++; + return _keys.TryGetValue(clusterId, out psk); + } + + /// + /// Overrides the interface's default so the two calls are separately countable. The + /// semantics are the default's: no key, or a key that is not exactly the presented token. + /// + public bool Matches(string clusterId, string presented) + { + MatchesCalls++; + return !string.IsNullOrWhiteSpace(presented) + && _keys.TryGetValue(clusterId, out var expected) + && string.Equals(expected, presented, StringComparison.Ordinal); + } + + /// No-op: the gate is the production provider's behaviour, pinned in its own tests. + public void SetKnownClusters(IReadOnlyCollection clusterIds) { } public void Invalidate(string clusterId) { } From 6a86201105a9ef17ba0d3efaf8fafc5479a41348 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 6 Aug 2026 05:58:21 -0400 Subject: [PATCH 04/12] feat(mesh): gRPC command fan-out behind Mode=Grpc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MeshGrpcCommandDialer places one NodeCommandService.Execute call on one node: a GrpcChannel cached per endpoint (the GrpcDeploymentArtifactFetcher shape), the TARGET node's cluster's bearer plus x-otopcua-cluster naming that cluster, and a per-call deadline linked to the caller's token. A cluster with no resolvable key is refused locally with Unauthenticated — there is no arm on which a command leaves central unauthenticated, even one the far side would have rejected. CentralCommunicationActor gains the third mode branch. The contact load now also projects GrpcPort and, from the SAME rows in the SAME pass, builds the dispatch set — which is what makes 'the gRPC fan reaches exactly the SendToAll recipient set' structural rather than two queries that have to keep agreeing. A row with a null GrpcPort is skipped warn-once (the TelemetryNodeSource precedent). Per-node results are piped back for LOGGING ONLY: named node, gRPC status, dropped. No retry, no queue. Every dial is scheduled on the thread pool rather than called from the receive. Found by review and pinned red-first: an async method runs synchronously to its first await, and the dialer's pre-await work is TryGetPsk, which blocks on the secret store — bounded but seconds, and repeated after every refresh since the refresh drops the cache. Dialing inline parked the actor's mailbox, so deployment acks queued behind a slow node; the regression test tells an ApplyAck while a dial is held and fails without the hop. The contact refresh also wires the Phase 1A register: it publishes the loaded cluster ids as the PSK provider's known-cluster set and drops the key cache, so a rotated key takes effect within one ContactRefreshSeconds. Neither happens under Dps or ClusterClient. Both existing mode branches are behaviourally untouched and every existing fact passes unmodified. The one edit to shared text: the Status.Failure warning drops the word 'ClusterClient', since that handler now serves all three transports. ControlPlane.Tests 205 -> 230, Host.Tests 92, Cluster.Tests 184, 0 warnings. --- .../CentralCommunicationActor.cs | 427 ++++++++++++- .../Communication/MeshGrpcCommandDialer.cs | 159 +++++ .../ServiceCollectionExtensions.cs | 13 +- src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs | 8 + .../CentralCommunicationActorTests.cs | 561 ++++++++++++++++++ .../MeshGrpcCommandDialerTests.cs | 271 +++++++++ 6 files changed, 1428 insertions(+), 11 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs index 0c470ede..635194ff 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs @@ -3,17 +3,24 @@ using Akka.Actor; using Akka.Cluster.Tools.Client; using Akka.Cluster.Tools.PublishSubscribe; using Akka.Event; +using Grpc.Core; using Microsoft.EntityFrameworkCore; using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; +using ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1; +using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Configuration; +// Grpc.Core (imported for RpcException on the dial path) declares a Status of its own. This file's +// bare Status is Akka's — the Status.Failure a faulted piped task arrives as; the gRPC one is only +// ever reached through an RpcException instance. +using Status = Akka.Actor.Status; namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; /// /// Central-side end of the central↔node command boundary (per-cluster mesh Phase 2). Routes -/// every central→node command out — over DistributedPubSub or ClusterClient, per +/// every central→node command out — over DistributedPubSub, ClusterClient or gRPC, per /// — and forwards inbound s to the /// deploy coordinator singleton. /// @@ -31,6 +38,23 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; /// and fans SendToAll across all of them. See — this is the /// single most important thing to understand before changing this actor. /// +/// +/// The gRPC branch fans over the same recipient set, node by node +/// (ClusterClient→gRPC Phase 1A). SendToAll reaches every node registered on a mesh's +/// receptionist, and the contact load already selects exactly the enabled, non-maintenance +/// rows those nodes come from; the gRPC branch dials that same row set directly, one call per +/// node, so the recipient set is identical by construction rather than by coincidence. It is +/// deliberately NOT narrower: the node side has no ClusterId or node filter at all — +/// scoping happens later, inside the artifact — so a node left out here silently keeps its +/// old configuration while the deployment seals green. +/// +/// +/// Fire-and-forget, on every transport. The gRPC branch pipes per-node results back to +/// itself for logging ONLY: a failure names the node and the status and is dropped. No retry, +/// no queue, no ordering guarantee across nodes. That is not an omission — it is the +/// ClusterClient semantic preserved exactly, and the backstops are unchanged (the deploy +/// coordinator's apply deadline, and a node's boot-time replay). +/// /// public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers { @@ -40,8 +64,11 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers private readonly MeshTransportOptions _options; private readonly IMeshClusterClientFactory _clientFactory; private readonly Func _coordinator; + private readonly IMeshCommandDialer? _dialer; + private readonly IMeshPskProvider? _pskProvider; private readonly ILoggingAdapter _log = Context.GetLogger(); private readonly bool _useClusterClient; + private readonly bool _useGrpc; // One client per application Cluster, keyed by ClusterId. A ClusterClient reaches only the mesh // whose receptionist it dialled, and after Phase 6 every cluster is its own mesh — so central must @@ -49,6 +76,15 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers private readonly Dictionary Contacts)> _clients = new(); + // Node ids already warned about for a null GrpcPort. A persistently portless row would otherwise + // produce one warning per refresh forever; same warn-once shape as TelemetryNodeSource. Actor + // thread only. + private readonly HashSet _warnedPortlessNodes = new(StringComparer.Ordinal); + + // The gRPC dispatch set, replaced wholesale on every contact load. Actor thread only; the fan + // captures the list reference, so a refresh mid-fan cannot mutate a dispatch already under way. + private IReadOnlyList _targets = []; + private CancellationTokenSource _lifecycle = new(); /// Gets the timer scheduler driving the periodic contact refresh. @@ -63,46 +99,73 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers /// an so this actor never depends on registration order at wiring /// time — it resolves on first use and caches. /// + /// + /// Places one command call on one node, under . + /// Optional so the two older transports need nothing new wired; a + /// dialer under Grpc is a mis-wiring and is reported loudly rather than silently + /// dropping every command. + /// + /// + /// Central's per-cluster key lookup, told on every contact refresh which clusters exist (the + /// known-cluster gate) and to drop its cache (so a rotated key takes effect within one + /// refresh). Optional for the same reason as . + /// /// The props. public static Props Props( IDbContextFactory dbFactory, MeshTransportOptions options, IMeshClusterClientFactory clientFactory, - Func coordinator) => + Func coordinator, + IMeshCommandDialer? dialer = null, + IMeshPskProvider? pskProvider = null) => Akka.Actor.Props.Create(() => - new CentralCommunicationActor(dbFactory, options, clientFactory, coordinator)); + new CentralCommunicationActor( + dbFactory, options, clientFactory, coordinator, dialer, pskProvider)); /// Initializes a new instance of the class. /// Factory for the config database. /// The bound mesh-transport options. /// Creates the ClusterClient. /// Lazily resolves the deploy-coordinator singleton proxy. + /// Places one gRPC command call on one node; Grpc mode only. + /// Central's per-cluster key lookup; Grpc mode only. public CentralCommunicationActor( IDbContextFactory dbFactory, MeshTransportOptions options, IMeshClusterClientFactory clientFactory, - Func coordinator) + Func coordinator, + IMeshCommandDialer? dialer = null, + IMeshPskProvider? pskProvider = null) { _dbFactory = dbFactory; _options = options; _clientFactory = clientFactory; _coordinator = coordinator; + _dialer = dialer; + _pskProvider = pskProvider; _useClusterClient = string.Equals( options.Mode, MeshTransportOptions.ModeClusterClient, StringComparison.OrdinalIgnoreCase); + _useGrpc = string.Equals( + options.Mode, MeshTransportOptions.ModeGrpc, StringComparison.OrdinalIgnoreCase); Receive(HandleMeshCommand); Receive(HandleApplyAck); Receive(_ => LoadContactsFromDb()); Receive(HandleContactsLoaded); + Receive(HandleCommandDispatched); // A faulted LoadContactsFromDb task is piped here as a Status.Failure. Without this handler // the failure is an unhandled message (debug level only) and the refresh fails silently — // an operator cannot distinguish "no nodes configured" from "the database is down", and the // contact set silently freezes at whatever it last held. + // + // Under Grpc that freeze extends to the known-cluster gate, deliberately: a failed load must + // neither widen the gate nor narrow it, and the previous set is the last thing central + // actually knew. Receive(f => _log.Warning( f.Cause, - "Failed to load ClusterNode contact points; the mesh ClusterClient contact set was NOT " - + "refreshed and may be stale or empty. Commands may be dropped until the next refresh")); + "Failed to load ClusterNode contact points; the mesh contact set was NOT refreshed and " + + "may be stale or empty. Commands may be dropped until the next refresh")); Receive(_ => { /* DPS subscribe confirmation */ }); } @@ -110,7 +173,7 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers /// protected override void PreStart() { - if (!_useClusterClient) + if (!_useClusterClient && !_useGrpc) { _log.Info( "Mesh transport is {Mode}: commands are published on DistributedPubSub and no " @@ -118,6 +181,19 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers return; } + if (_useGrpc && _dialer is null) + { + // Not a fatal throw: the ack path (HandleApplyAck) is transport-independent and still + // works, so taking the actor down would turn a command-plane mis-wiring into an ack + // outage as well. Every dispatch logs its own error, so this cannot go unnoticed. + _log.Error( + "Mesh transport is Grpc but no command dialer was supplied to this actor; EVERY " + + "central→node command will be dropped. Check the ControlPlane registration wires " + + "IMeshCommandDialer"); + } + + // The refresh drives more than ClusterClient contacts under Grpc: it is also what publishes + // the dispatch target set and keeps the PSK provider's known-cluster gate current. Timers.StartPeriodicTimer( RefreshTimerKey, new RefreshContacts(), @@ -131,12 +207,23 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers foreach (var (client, _) in _clients.Values) Context.Stop(client); _clients.Clear(); + // Cancelling the lifecycle token also cancels every gRPC dial in flight. The dialer itself is + // deliberately NOT disposed here: it is a container-owned singleton shared with anything else + // that may dial, and an actor restart must not tear its channel pool down. _lifecycle.Cancel(); _lifecycle.Dispose(); } private void HandleMeshCommand(MeshCommand cmd) { + // Checked before the DPS arm: _useClusterClient is false under Grpc too, so the order is + // what keeps a Grpc-mode command off DistributedPubSub. + if (_useGrpc) + { + DispatchOverGrpc(cmd); + return; + } + if (!_useClusterClient) { DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(cmd.Topic, cmd.Message)); @@ -168,6 +255,199 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers client.Tell(new ClusterClient.SendToAll(MeshPaths.NodeCommunication, cmd.Message)); } + /// + /// Encodes one command once and fans it to every target node, one unary gRPC call each. + /// + /// The command envelope carrying the domain record. + /// + /// + /// Encoded once, before the fan. Not per node — partly because the wire form is + /// identical for every recipient, but mostly because AlarmCommand has no + /// correlation member of its own and the envelope mints one: minting inside the loop + /// would give one operator action N unrelated correlation ids and make it untraceable. + /// + /// + /// Nothing captured from the actor. Each dial is a static call taking only locals, + /// so no per-node continuation can touch actor state off the actor's thread. Results come + /// back as ordinary messages via PipeTo and are handled on the actor thread like + /// anything else. + /// + /// + /// Every dial starts on the thread pool, not here. An async method runs + /// synchronously up to its first await, and the dialer's pre-await work includes + /// IMeshPskProvider.TryGetPsk, which blocks on the secret store (bounded, but + /// seconds) the first time a cluster is looked up — and again after every refresh, since + /// the refresh drops the cache to pick up rotations. Calling straight into it would park + /// this actor's mailbox, and deployment acks queue behind that mailbox. + /// + /// + private void DispatchOverGrpc(MeshCommand cmd) + { + var commandType = cmd.Message.GetType().Name; + + var dialer = _dialer; + if (dialer is null) + { + _log.Error( + "Mesh transport is Grpc but no command dialer is wired — dropping {MessageType}. The " + + "command is not buffered and will not be retried", + commandType); + return; + } + + // Captured before the fan: a contact refresh between here and the last dial must not change + // the recipient set of a dispatch already under way. + var targets = _targets; + if (targets.Count == 0) + { + // App-level drop, exactly as the ClusterClient arm does with no client. The command is + // NOT queued and will NOT be retried: a deploy fails at the coordinator's apply deadline + // naming the silent nodes, which is a better operator signal than a command arriving + // minutes late against a config the DB already recorded as failed. + _log.Warning( + "No mesh gRPC targets — dropping {MessageType}. The last contact refresh produced no " + + "enabled, non-maintenance ClusterNode row carrying a GrpcPort in any cluster; the " + + "command is not buffered and will not be retried", + commandType); + return; + } + + CommandEnvelope envelope; + try + { + envelope = MeshTransportDtoMapper.ToEnvelope(cmd.Message, new CorrelationId(Guid.NewGuid())); + } + catch (Exception ex) + { + // The mapper refuses a payload the far side could not decode. Dropping here is the same + // outcome the old transport gave for such a message (nothing on the node matched it), but + // loud instead of silent. + _log.Error( + ex, + "Cannot encode {MessageType} for the gRPC command plane; the command is dropped and " + + "was NOT delivered to any node", + commandType); + return; + } + + CancellationToken ct; + try + { + ct = _lifecycle.Token; + } + catch (ObjectDisposedException) + { + return; // Stopping. + } + + var self = Self; + var correlationId = envelope.CorrelationId; + foreach (var target in targets) + { + var node = target; + + // CancellationToken.None on Task.Run, deliberately: cancelling the SCHEDULING would make + // the task complete as canceled, which PipeTo converts into a Status.Failure — the one + // message this actor already reads as "the contact load failed". The lifecycle token is + // passed into the dial instead, where it cancels the call and comes back as an ordinary + // per-node outcome. + Task.Run( + () => DispatchOneAsync(dialer, node, envelope, commandType, correlationId, ct), + CancellationToken.None) + .PipeTo(self); + } + } + + /// + /// Dials one node and turns every possible outcome into a . + /// + /// The dialer seam. + /// The node to dial. + /// The already-encoded command. + /// CLR type name of the command, for the log. + /// Transport correlation, for the log. + /// The actor's lifecycle token. + /// The outcome, always — this method does not fault. + /// + /// Catching everything is what keeps the result on ONE channel. A faulting task piped to an + /// actor arrives as Status.Failure, which this actor already handles as "the contact + /// load failed" — so an unhandled dial fault would log a database-outage warning about a node + /// that simply did not answer. + /// + private static async Task DispatchOneAsync( + IMeshCommandDialer dialer, + MeshNodeTarget target, + CommandEnvelope envelope, + string commandType, + string correlationId, + CancellationToken ct) + { + try + { + await dialer.DispatchAsync(target.Endpoint, envelope, target.ClusterId, ct) + .ConfigureAwait(false); + return new CommandDispatched(target.NodeId, commandType, correlationId, null, null); + } + catch (RpcException ex) + { + return new CommandDispatched( + target.NodeId, + commandType, + correlationId, + ex.StatusCode.ToString(), + Truncate(ex.Status.Detail)); + } + catch (OperationCanceledException) + { + // Named rather than swept into the generic arm: this is the ordinary shape of a node that + // did not answer in time, and "Unknown / TaskCanceledException" would send an operator + // looking for a bug instead of at DispatchTimeoutSeconds. + return new CommandDispatched( + target.NodeId, + commandType, + correlationId, + "Cancelled", + "the dial did not complete within MeshTransport:DispatchTimeoutSeconds, or central " + + "is shutting down"); + } + catch (Exception ex) + { + // Type name only, never the message: this path also covers whatever the dialer's channel + // construction can throw, and a message is the one place a misconfigured endpoint string + // could carry something an operator did not mean to put in a log. + return new CommandDispatched( + target.NodeId, commandType, correlationId, "Unknown", ex.GetType().Name); + } + } + + /// + /// Bounds a status detail before it reaches the log. The string comes from the far side, and + /// the far side is a node — trusted, but not a reason to let it choose how many lines of + /// central's log one failed command occupies. + /// + /// The received status detail. + /// The detail, at most 200 characters. + private static string Truncate(string? detail) + => detail is null || detail.Length <= 200 ? detail ?? "" : detail[..200] + "…"; + + private void HandleCommandDispatched(CommandDispatched result) + { + if (result.FailureStatus is null) + { + _log.Debug( + "Mesh: {CommandType} (correlationId={CorrelationId}) accepted by {NodeId} over gRPC", + result.CommandType, result.CorrelationId, result.NodeId); + return; + } + + _log.Warning( + "Mesh: {CommandType} (correlationId={CorrelationId}) was NOT delivered to {NodeId} " + + "({Status}: {Detail}). It is dropped and will NOT be retried — the deployment apply " + + "deadline and the node's boot-time replay are the backstops", + result.CommandType, result.CorrelationId, result.NodeId, + result.FailureStatus, result.FailureDetail); + } + private void HandleApplyAck(ApplyAck ack) { var coordinator = _coordinator(); @@ -210,7 +490,7 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers var rows = await db.ClusterNodes .AsNoTracking() .Where(n => n.Enabled && !n.MaintenanceMode) - .Select(n => new { n.NodeId, n.ClusterId, n.Host, n.AkkaPort }) + .Select(n => new { n.NodeId, n.ClusterId, n.Host, n.AkkaPort, n.GrpcPort }) .ToListAsync(ct) .ConfigureAwait(false); @@ -219,8 +499,28 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers // client. var byCluster = new Dictionary>(); var malformed = new List(); + + // The gRPC dispatch set, built from the SAME row set in the SAME pass. Sharing the read is + // what makes "the gRPC fan reaches exactly the ClusterClient recipient set" structural + // rather than a claim two separate queries would have to keep agreeing on. + var targets = new List(rows.Count); + var portless = new List(); + + // Every cluster with at least one enabled, non-maintenance row — the known-cluster gate's + // membership set. Deliberately independent of GrpcPort: the gate answers "is this a cluster + // central knows", and a cluster central cannot currently dial can still send it an ack. + var clusterIds = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var row in rows) { + clusterIds.Add(row.ClusterId); + + if (row.GrpcPort is { } grpcPort) + targets.Add(new MeshNodeTarget( + row.NodeId, row.ClusterId, $"http://{row.Host}:{grpcPort}")); + else + portless.Add(row.NodeId); + var address = $"akka.tcp://{systemName}@{row.Host}:{row.AkkaPort}/system/receptionist"; // Parse up front, per row, inside the loop's own guard. A single malformed row must @@ -241,12 +541,19 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers var contactsByCluster = byCluster.ToDictionary( kv => kv.Key, kv => (IReadOnlyList)kv.Value); - return new ContactsLoaded(contactsByCluster, malformed); + return new ContactsLoaded( + contactsByCluster, malformed, targets, portless, clusterIds.ToList()); }, ct).PipeTo(self); } private void HandleContactsLoaded(ContactsLoaded msg) { + if (_useGrpc) + { + HandleGrpcContactsLoaded(msg); + return; + } + foreach (var bad in msg.Malformed) { _log.Warning( @@ -305,6 +612,59 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers } } + /// + /// Adopts a contact load under : the dispatch + /// target set, and the two things the PSK provider needs kept current. + /// + /// The load result. + /// + /// + /// The malformed-receptionist-address warnings are deliberately NOT logged here: nothing + /// on this transport dials a receptionist, and reporting an unusable Akka address as a + /// command-plane problem would send an operator after the wrong column. A host that is + /// unusable as a gRPC endpoint surfaces instead as a named per-node dial failure on every + /// dispatch, which is a truer signal. + /// + /// + /// InvalidateAll, not per-cluster Invalidate. The requirement is that + /// a rotated key takes effect within one refresh, and only a whole-cache drop delivers + /// that: a key can rotate without the cluster's membership changing at all, so invalidating + /// only departed clusters would serve the pre-rotation key forever. The cost is one store + /// read per cluster per refresh — the cache holds a single entry per cluster — and it is + /// paid only by clusters that are actually dialled. + /// + /// + private void HandleGrpcContactsLoaded(ContactsLoaded msg) + { + foreach (var nodeId in msg.PortlessNodes) + { + // Warn-once per node id: a persistently portless row is re-skipped every refresh, and the + // same repetition problem TelemetryNodeSource solved this way. + if (_warnedPortlessNodes.Add(nodeId)) + { + _log.Warning( + "ClusterNode {NodeId} has no GrpcPort; it exposes no command endpoint and is " + + "skipped in this dispatch set (further skips of this node are silent). Commands " + + "will not reach it until the row carries a port", nodeId); + } + } + + _targets = msg.Targets; + + // Published AFTER the target set is adopted, so anything observing the gate can rely on the + // dispatch set already being current. + _pskProvider?.SetKnownClusters(msg.ClusterIds); + _pskProvider?.InvalidateAll(); + + if (_targets.Count == 0) + { + _log.Warning( + "No enabled, non-maintenance ClusterNode row carries a GrpcPort in any cluster; there " + + "are no gRPC command targets and every central→node command will be dropped until a " + + "refresh finds one"); + } + } + /// Stops a cluster's existing client and creates a replacement for its new contact set. /// /// @@ -371,7 +731,54 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers /// Cluster's id. One ClusterClient is built per key. /// /// Rows that did not parse, described for the log. + /// + /// The gRPC dispatch set: one entry per enabled, non-maintenance row that carries a + /// GrpcPort. Built from the same rows as . + /// + /// + /// Node ids skipped because their row has no GrpcPort. Carried rather than logged in + /// the load task so the warn-once bookkeeping stays on the actor's thread. + /// + /// + /// Every application Cluster with at least one enabled, non-maintenance row — + /// the PSK provider's known-cluster gate membership. + /// public sealed record ContactsLoaded( IReadOnlyDictionary> ContactsByCluster, - IReadOnlyList Malformed); + IReadOnlyList Malformed, + IReadOnlyList Targets, + IReadOnlyList PortlessNodes, + IReadOnlyList ClusterIds); + + /// One node the gRPC command plane can dial. + /// The ClusterNode row's id — what a failure log names. + /// + /// The node's application Cluster. Selects the pre-shared key presented to it: a node + /// dialled with another cluster's id is graded against the wrong key and refuses the command. + /// + /// The h2c base address, http://{Host}:{GrpcPort}. + public sealed record MeshNodeTarget(string NodeId, string ClusterId, string Endpoint); + + /// + /// The outcome of one node's dial, piped back to the actor for logging only. Nothing + /// retries, counts or waits on these — a deployment's completion is judged by the acks nodes + /// send back, never by whether central's outbound call succeeded. + /// + /// The node dialled. + /// CLR type name of the command. + /// The envelope's transport correlation. + /// + /// on success; otherwise the gRPC status code's name, or + /// Unknown for a non-gRPC fault. + /// + /// + /// The gRPC status detail, or the exception's type name for a non-gRPC fault. Never a key, + /// and never a raw exception message. + /// + public sealed record CommandDispatched( + string NodeId, + string CommandType, + string CorrelationId, + string? FailureStatus, + string? FailureDetail); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs new file mode 100644 index 00000000..46dcd830 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs @@ -0,0 +1,159 @@ +using System.Collections.Concurrent; +using Grpc.Core; +using Grpc.Net.Client; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; + +/// +/// Places one central→node command call on one node's NodeCommandService +/// (ClusterClient→gRPC migration Phase 1A). The seam CentralCommunicationActor fans +/// across its target set, kept as an interface so the actor's tests never open a socket. +/// +/// +/// One dial, one node, no retry and no fan-out of its own — the fan and its per-node failure +/// logging belong to the actor, which is the component that knows what the recipient set is. +/// +public interface IMeshCommandDialer +{ + /// Delivers one command envelope to one node. + /// The node's gRPC base address, e.g. http://site-a-1:4056. + /// The wire envelope, already mapped from the domain record. + /// + /// The application Cluster the target node belongs to. Selects which pre-shared + /// key is presented, and is echoed to the node in x-otopcua-cluster. + /// + /// Cancels the dial; a per-call deadline is applied on top of it. + /// A task completing when the node has accepted the command. + /// + /// The node rejected the call, was unreachable, or the deadline expired — and also when no + /// key could be established for , which is refused locally with + /// rather than dialled unauthenticated. + /// + Task DispatchAsync(string nodeEndpoint, CommandEnvelope envelope, string clusterId, CancellationToken ct); +} + +/// +/// +/// +/// Fail-closed on a missing key, always. There is no arm on which a command leaves +/// central without the target cluster's bearer. A node's MeshCommandAuthInterceptor +/// would reject an unauthenticated call anyway, but "the far side would have refused it" is +/// not a reason to make the call — an unauthenticated dial is a credential-free command +/// attempt against a node's control surface, and it should not exist even as a rejected one. +/// The refusal is reported as an so the actor's per-node failure +/// logging covers it on the same channel as a remote rejection; the operator response is +/// identical either way (seed the cluster's key), and a second channel would need a second +/// handler that could drift. +/// +/// +/// One channel per endpoint, cached. Same shape as +/// GrpcDeploymentArtifactFetcher: a owns a connection pool +/// and creating one per call leaks sockets. The cache is keyed by the endpoint string +/// ordinally — endpoints come from ClusterNode.Host + GrpcPort, so two spellings +/// of one host would merely cost a second channel, never a wrong destination. +/// +/// +/// The deadline is per call, not per fan. A dispatch dials every node in the fleet; +/// an unbounded call to one unreachable node would hold a task and a socket indefinitely +/// while the deploy path's own 2-minute apply deadline ticked down. +/// bounds it, linked to the +/// caller's token so actor shutdown still cancels everything in flight. +/// +/// +/// Registered as a singleton and disposed by the container. Disposal is one-way: a dial +/// after it throws rather than silently repopulating the cache with channels nobody will +/// ever dispose. +/// +/// +public sealed class MeshGrpcCommandDialer : IMeshCommandDialer, IDisposable +{ + /// + /// Header naming the cluster the presented bearer belongs to. Central's own inbound gate + /// reads the same header on the ack path; sending it outbound keeps one convention and tells + /// a node which key central graded itself against. + /// + private const string ClusterHeader = "x-otopcua-cluster"; + + private readonly IMeshPskProvider _pskProvider; + private readonly int _dispatchTimeoutSeconds; + private readonly Func _clientFactory; + private readonly ConcurrentDictionary _channels = new(StringComparer.Ordinal); + + private volatile bool _disposed; + + /// Initializes a new instance of the class. + /// Supplies the bearer for the target node's cluster. + /// Mesh transport options; supplies the per-call deadline. + /// + /// Seam for tests: builds the generated client for an endpoint. Defaults to a real h2c + /// , one cached per endpoint. + /// + public MeshGrpcCommandDialer( + IMeshPskProvider pskProvider, + IOptions options, + Func? clientFactory = null) + { + ArgumentNullException.ThrowIfNull(pskProvider); + ArgumentNullException.ThrowIfNull(options); + + _pskProvider = pskProvider; + _dispatchTimeoutSeconds = options.Value.DispatchTimeoutSeconds; + _clientFactory = clientFactory ?? DefaultClientFactory; + } + + /// + public async Task DispatchAsync( + string nodeEndpoint, CommandEnvelope envelope, string clusterId, CancellationToken ct) + { + ArgumentException.ThrowIfNullOrWhiteSpace(nodeEndpoint); + ArgumentNullException.ThrowIfNull(envelope); + ArgumentException.ThrowIfNullOrWhiteSpace(clusterId); + ObjectDisposedException.ThrowIf(_disposed, this); + + if (!_pskProvider.TryGetPsk(clusterId, out var psk)) + { + // The cluster id is operator-authored configuration, not a secret, so naming it is what + // makes this actionable. The key never appears here or anywhere else in this class. + throw new RpcException(new Status( + StatusCode.Unauthenticated, + $"No mesh pre-shared key could be established for cluster '{clusterId}'; the dial was " + + "refused locally rather than made unauthenticated. Seed the secret " + + "OT-MESH-PSK- on this node, or set MeshTransport:ClusterPsks: " + + "on a rig.")); + } + + var headers = new Metadata + { + { "authorization", $"Bearer {psk}" }, + { ClusterHeader, clusterId }, + }; + + using var deadlineCts = CancellationTokenSource.CreateLinkedTokenSource(ct); + if (_dispatchTimeoutSeconds > 0) + deadlineCts.CancelAfter(TimeSpan.FromSeconds(_dispatchTimeoutSeconds)); + + var client = _clientFactory(nodeEndpoint); + using var call = client.ExecuteAsync(envelope, headers, cancellationToken: deadlineCts.Token); + await call.ResponseAsync.ConfigureAwait(false); + } + + /// + public void Dispose() + { + _disposed = true; + + foreach (var channel in _channels.Values) + channel.Dispose(); + _channels.Clear(); + } + + private NodeCommandService.NodeCommandServiceClient DefaultClientFactory(string endpoint) + { + // h2c: GrpcChannel over an http:// address negotiates prior-knowledge HTTP/2. + var channel = _channels.GetOrAdd(endpoint, GrpcChannel.ForAddress); + return new NodeCommandService.NodeCommandServiceClient(channel); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs index 08cf21e9..32a04231 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs @@ -66,6 +66,15 @@ public static class ServiceCollectionExtensions var dbFactory = resolver.GetService>(); var options = resolver.GetService>().Value; + // Through IServiceProvider rather than the Akka resolver, deliberately: these two are + // OPTIONAL (Grpc mode only, and admin-role only within that), and GetService's + // null-for-absent contract is the thing being relied on. The Akka resolver's + // GetService makes no such promise, and this same block runs in test harnesses whose + // service collections register neither. + var serviceProvider = resolver.GetService(); + var dialer = serviceProvider.GetService(); + var pskProvider = serviceProvider.GetService(); + var actor = system.ActorOf( CentralCommunicationActor.Props( dbFactory, @@ -75,7 +84,9 @@ public static class ServiceCollectionExtensions // which Akka.Hosting materialises the singleton proxy relative to this block. () => registry.TryGet(out var coordinator) ? coordinator - : null), + : null, + dialer, + pskProvider), MeshPaths.CentralCommunicationName); // Registered unconditionally, in both transport modes. Under Dps nothing dials it, and diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index 33ff5a43..2fb407a3 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -534,6 +534,14 @@ if (hasAdmin) // Depends on ISecretResolver, which AddOtOpcUaSecrets registers unconditionally below (DI // registration order is irrelevant; resolution order is not). builder.Services.AddSingleton(); + // Central's outbound command dial (ClusterClient→gRPC Phase 1A). Admin-only for the same reason + // as the provider above — a node never dials another node — and singleton because it owns a + // GrpcChannel per node endpoint; per-call channels would leak sockets across a fleet-wide fan. + // Registered in every mode: CentralCommunicationActor only reaches it under Mode=Grpc, and an + // idle registration costs nothing, whereas gating it here would mean flipping the mode on a + // running fleet needs a central restart before any command lands — the same reasoning that + // registers the ClusterClient receptionist unconditionally. + builder.Services.AddSingleton(); } // Registered unconditionally: driver-role nodes resolve Layer-B DriverConfig secrets and have no auth/DP/AdminUI. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs index 97e3f28f..8563f781 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs @@ -1,4 +1,5 @@ using System.Collections.Immutable; +using System.Diagnostics.CodeAnalysis; using Akka.Actor; using Akka.Cluster.Tools.Client; using Akka.Cluster.Tools.PublishSubscribe; @@ -9,6 +10,8 @@ using Xunit; using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; @@ -56,6 +59,13 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase ContactRefreshSeconds = refreshSeconds, }; + private static MeshTransportOptions GrpcMode(int refreshSeconds = 60) => new() + { + Mode = MeshTransportOptions.ModeGrpc, + ContactRefreshSeconds = refreshSeconds, + DispatchTimeoutSeconds = 10, + }; + private static void SeedCluster( IDbContextFactory factory, string clusterId, @@ -92,6 +102,46 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase db.SaveChanges(); } + /// + /// Seeds a cluster whose nodes carry a GrpcPort — the gRPC command plane's shape. + /// Kept separate from so the ClusterClient facts, which have no + /// opinion about the port, stay exactly as they were. + /// + private static void SeedGrpcCluster( + IDbContextFactory factory, + string clusterId, + params (string NodeId, string Host, int? GrpcPort, bool Enabled, bool Maintenance)[] nodes) + { + using var db = factory.CreateDbContext(); + db.ServerClusters.Add(new ServerCluster + { + ClusterId = clusterId, + Name = clusterId, + Enterprise = "ZB", + Site = "Site", + RedundancyMode = RedundancyMode.None, + NodeCount = 1, + CreatedBy = "test", + }); + foreach (var n in nodes) + { + db.ClusterNodes.Add(new ClusterNode + { + NodeId = n.NodeId, + ClusterId = clusterId, + Host = n.Host, + AkkaPort = 4053, + GrpcPort = n.GrpcPort, + ApplicationUri = $"urn:{n.NodeId}", + Enabled = n.Enabled, + MaintenanceMode = n.Maintenance, + CreatedBy = "test", + }); + } + + db.SaveChanges(); + } + private IActorRef Spawn( IDbContextFactory factory, MeshTransportOptions options, @@ -306,4 +356,515 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase actor.Tell(new MeshCommand(DeploymentsTopic, "anything")); ExpectNoMsg(TimeSpan.FromMilliseconds(200)); } + + // ------------------------------------------------------------------ Mode = Grpc + // + // The third transport. The fan-out must reach EXACTLY the recipient set SendToAll reaches today + // — every enabled, non-maintenance node in every cluster — because the node side still has no + // ClusterId or node filter; scoping happens later, inside the artifact. A narrower fan would + // leave nodes silently on their old configuration while the deployment sealed green. + + [Fact] + public void Grpc_mode_fans_one_command_to_every_enabled_node_across_all_clusters() + { + // THE assertion of this task, and the direct counterpart of + // A_command_fans_out_a_send_to_all_to_every_cluster_client. + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster( + factory, + "C1", + ("host-a:4053", "host-a", 4056, true, false), + ("host-b:4053", "host-b", 4056, true, false)); + SeedGrpcCluster(factory, "C2", ("host-c:4053", "host-c", 4056, true, false)); + var rig = SpawnGrpcRig(factory); + AwaitContactsLoaded(rig); + + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch())); + + AwaitCondition(() => rig.Dialer.Calls.Count == 3, TimeSpan.FromSeconds(5)); + rig.Dialer.Calls.Select(c => c.Endpoint).ShouldBe( + new[] { "http://host-a:4056", "http://host-b:4056", "http://host-c:4056" }, ignoreOrder: true); + + // Each node is dialled with ITS OWN cluster's id — that is what selects the pre-shared key, + // so getting it wrong turns every command into a PermissionDenied at the node. + rig.Dialer.Calls.Single(c => c.Endpoint.Contains("host-c")).ClusterId.ShouldBe("C2"); + rig.Dialer.Calls.Where(c => !c.Endpoint.Contains("host-c")).ShouldAllBe(c => c.ClusterId == "C1"); + } + + [Fact] + public void Grpc_mode_creates_no_ClusterClient_and_publishes_nothing_on_pubsub() + { + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false)); + var clients = new RecordingClientFactory(() => CreateTestProbe()); + var mediatorProbe = CreateTestProbe(); + DistributedPubSub.Get(Sys).Mediator.Tell( + new Subscribe(DeploymentsTopic, mediatorProbe.Ref), mediatorProbe.Ref); + mediatorProbe.ExpectMsg(TimeSpan.FromSeconds(5)); + + var rig = SpawnGrpcRig(factory, clientFactory: clients); + AwaitContactsLoaded(rig); + + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch())); + + AwaitCondition(() => rig.Dialer.Calls.Count == 1, TimeSpan.FromSeconds(5)); + clients.Calls.ShouldBeEmpty(); + mediatorProbe.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + } + + [Fact] + public void Grpc_mode_does_not_dial_disabled_or_maintenance_nodes() + { + // Same recipient filter as the ClusterClient contact load, asserted independently so the two + // transports cannot drift on who counts as a node. + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster( + factory, + "C1", + ("host-a:4053", "host-a", 4056, true, false), + ("host-b:4053", "host-b", 4056, false, false), + ("host-c:4053", "host-c", 4056, true, true)); + var rig = SpawnGrpcRig(factory); + AwaitContactsLoaded(rig); + + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch())); + + AwaitCondition(() => rig.Dialer.Calls.Count == 1, TimeSpan.FromSeconds(5)); + rig.Dialer.Calls[0].Endpoint.ShouldContain("host-a"); + + // And no late arrival from the two excluded rows. + ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + rig.Dialer.Calls.Count.ShouldBe(1); + } + + [Fact] + public void A_node_with_no_GrpcPort_is_skipped_with_a_warning_and_the_others_are_still_dialled() + { + // Mirrors TelemetryNodeSource: a row with no port exposes no endpoint yet. It must cost that + // node its commands, never the whole fan. + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster( + factory, + "C1", + ("host-a:4053", "host-a", null, true, false), + ("host-b:4053", "host-b", 4056, true, false)); + + GrpcRig? rig = null; + EventFilter.Warning(contains: "has no GrpcPort").ExpectOne( + TimeSpan.FromSeconds(5), () => rig = SpawnGrpcRig(factory)); + + rig.ShouldNotBeNull(); + AwaitContactsLoaded(rig); + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch())); + + AwaitCondition(() => rig.Dialer.Calls.Count == 1, TimeSpan.FromSeconds(5)); + rig.Dialer.Calls[0].Endpoint.ShouldBe("http://host-b:4056"); + } + + [Fact] + public void One_failing_node_does_not_stop_the_fan_and_is_logged_by_node_id() + { + // Fire-and-forget: a failure is logged and DROPPED. No retry, no queue, and above all no + // abort of the remaining nodes — the deploy path's apply deadline is the backstop. + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster( + factory, + "C1", + ("host-a:4053", "host-a", 4056, true, false), + ("host-b:4053", "host-b", 4056, true, false)); + var dialer = new RecordingDialer + { + FailEndpointsContaining = "host-a", + Fault = new Grpc.Core.RpcException( + new Grpc.Core.Status(Grpc.Core.StatusCode.Unavailable, "node down")), + }; + var rig = SpawnGrpcRig(factory, dialer); + AwaitContactsLoaded(rig); + + // The node id, not the endpoint: an operator reading the log is looking at ClusterNode rows. + EventFilter.Warning(contains: "host-a:4053").ExpectOne(TimeSpan.FromSeconds(5), () => + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch()))); + + // Both were attempted; the healthy one is unaffected by its neighbour's failure. + AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5)); + + // Still serving after a per-node failure — no supervision restart, no state loss. + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch())); + AwaitCondition(() => rig.Dialer.Calls.Count == 4, TimeSpan.FromSeconds(5)); + } + + [Fact] + public void A_psk_miss_costs_only_that_node_and_never_becomes_a_fan_level_abort() + { + // The dialer refuses a keyless cluster locally, so it arrives here as an ordinary per-node + // failure. The fan-level assertion is that the other cluster is still dialled. + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false)); + SeedGrpcCluster(factory, "C2", ("host-b:4053", "host-b", 4056, true, false)); + var dialer = new RecordingDialer + { + FailClusters = ["C2"], + Fault = new Grpc.Core.RpcException( + new Grpc.Core.Status(Grpc.Core.StatusCode.Unauthenticated, "no key")), + }; + var rig = SpawnGrpcRig(factory, dialer); + AwaitContactsLoaded(rig); + + EventFilter.Warning(contains: "Unauthenticated").ExpectOne(TimeSpan.FromSeconds(5), () => + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch()))); + + AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5)); + } + + [Fact] + public void A_command_with_no_grpc_target_is_dropped_with_a_warning() + { + var factory = NewInMemoryDbFactory(); + var rig = SpawnGrpcRig(factory); + AwaitContactsLoaded(rig); + + EventFilter.Warning(contains: "dropping").ExpectOne(TimeSpan.FromSeconds(5), () => + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch()))); + + rig.Dialer.Calls.ShouldBeEmpty(); + } + + [Fact] + public void A_payload_that_is_not_a_mesh_command_record_is_dropped_with_an_error() + { + // The mapper refuses to encode an unknown payload rather than shipping something the node + // cannot decode. That refusal must cost the command, not the actor. + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false)); + var rig = SpawnGrpcRig(factory); + AwaitContactsLoaded(rig); + + EventFilter.Error(contains: "Cannot encode").ExpectOne(TimeSpan.FromSeconds(5), () => + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, "not a command record"))); + + rig.Dialer.Calls.ShouldBeEmpty(); + + // Still serving. + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch())); + AwaitCondition(() => rig.Dialer.Calls.Count == 1, TimeSpan.FromSeconds(5)); + } + + [Fact] + public void The_envelope_carries_the_commands_own_correlation_and_the_whole_fan_shares_it() + { + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster( + factory, + "C1", + ("host-a:4053", "host-a", 4056, true, false), + ("host-b:4053", "host-b", 4056, true, false)); + var rig = SpawnGrpcRig(factory); + AwaitContactsLoaded(rig); + + var correlationId = new CorrelationId(Guid.NewGuid()); + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, new DispatchDeployment( + new DeploymentId(Guid.NewGuid()), new RevisionHash("abc"), correlationId))); + + AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5)); + rig.Dialer.Calls.ShouldAllBe(c => c.Envelope.CorrelationId == correlationId.ToString()); + // One envelope object, encoded once and fanned — not re-encoded per node. + rig.Dialer.Calls.Select(c => c.Envelope).Distinct().Count().ShouldBe(1); + } + + [Fact] + public void An_alarm_command_gets_one_minted_transport_correlation_shared_by_the_whole_fan() + { + // AlarmCommand is the one record with no correlation member of its own, so the envelope's is + // the only handle an operator has on it. Minting per fan rather than per node is what makes + // "this alarm command" one traceable event instead of N unrelated ones. + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster( + factory, + "C1", + ("host-a:4053", "host-a", 4056, true, false), + ("host-b:4053", "host-b", 4056, true, false)); + var rig = SpawnGrpcRig(factory); + AwaitContactsLoaded(rig); + + rig.Actor.Tell(new MeshCommand( + "alarms", new AlarmCommand("alarm-1", "Acknowledge", "operator", null, null))); + + AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5)); + rig.Dialer.Calls.Select(c => c.Envelope.CorrelationId).Distinct().Count().ShouldBe(1); + rig.Dialer.Calls[0].Envelope.CorrelationId.ShouldNotBeNullOrWhiteSpace(); + rig.Dialer.Calls[0].Envelope.AlarmCommand.AlarmId.ShouldBe("alarm-1"); + } + + [Fact] + public void The_contact_refresh_publishes_the_known_cluster_set_and_drops_cached_psks() + { + // Phase 1A security register item 1, wired: the gate is only a control if something keeps it + // current, and the refresh is the moment central's picture of the fleet is up to date. The + // InvalidateAll on the same tick is what makes a rotated key take effect within one refresh + // instead of requiring a central restart. + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false)); + SeedGrpcCluster(factory, "C2", ("host-b:4053", "host-b", null, true, false)); + var rig = SpawnGrpcRig(factory); + + AwaitContactsLoaded(rig); + + // C2 is included even though its only node has no GrpcPort: the gate is about which clusters + // central KNOWS, not which it can currently dial. Excluding it would deny that cluster's acks. + rig.Psk.KnownClusterPublications[0].ShouldBe(new[] { "C1", "C2" }, ignoreOrder: true); + rig.Psk.InvalidateAllCount.ShouldBeGreaterThanOrEqualTo(1); + } + + [Fact] + public void A_slow_dial_does_not_park_the_actors_mailbox() + { + // The dialer's first act is a PSK lookup, which blocks on the secret store — bounded, but + // seconds, and repeated after every refresh because the refresh drops the cache. An async + // method runs synchronously to its first await, so dialing straight from the receive would + // stall this actor for that long PER NODE, and deployment acks queue behind that mailbox. + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false)); + var coordinator = CreateTestProbe(); + var dialer = new RecordingDialer { BlockBeforeFirstAwait = true }; + var rig = SpawnGrpcRig(factory, dialer, coordinator: coordinator.Ref); + AwaitContactsLoaded(rig); + + try + { + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch())); + + // The ack must be handled while the dial is still parked. + var ack = new ApplyAck( + new DeploymentId(Guid.NewGuid()), + new NodeId("host-a:4053"), + ApplyAckOutcome.Applied, + null, + new CorrelationId(Guid.NewGuid())); + rig.Actor.Tell(ack); + + coordinator.ExpectMsg(TimeSpan.FromSeconds(5)).ShouldBe(ack); + } + finally + { + dialer.Release(); + } + } + + [Fact] + public void A_node_added_after_start_joins_the_fan_on_the_next_refresh() + { + // The refresh is not decoration: operators add and retire nodes, and central must pick them + // up without a restart. Under Grpc that is the ONLY thing keeping the dispatch set current — + // there is no receptionist to notice a new registration on its own. + // + // Driven by telling the actor its own RefreshContacts tick rather than by shortening + // ContactRefreshSeconds: a one-second timer would run for the whole fact, adding DB load to + // every test sharing the runner, and would make this assertion depend on wall-clock timing. + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false)); + var rig = SpawnGrpcRig(factory); + AwaitContactsLoaded(rig); + + using (var db = factory.CreateDbContext()) + { + db.ClusterNodes.Add(new ClusterNode + { + NodeId = "host-b:4053", + ClusterId = "C1", + Host = "host-b", + AkkaPort = 4053, + GrpcPort = 4056, + ApplicationUri = "urn:host-b", + Enabled = true, + CreatedBy = "test", + }); + db.SaveChanges(); + } + + rig.Actor.Tell(new CentralCommunicationActor.RefreshContacts()); + AwaitCondition( + () => rig.Psk.KnownClusterPublications.Count >= 2, TimeSpan.FromSeconds(5)); + + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch())); + + AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5)); + rig.Dialer.Calls.Select(c => c.Endpoint).ShouldBe( + new[] { "http://host-a:4056", "http://host-b:4056" }, ignoreOrder: true); + } + + [Fact] + public void The_known_cluster_set_is_not_published_by_the_other_two_modes() + { + // The gate is a gRPC-command-plane control. Publishing it under Dps or ClusterClient would + // start refusing acks on a transport whose ack path never consults the provider at all — + // and, worse, would do so from a code path those modes' tests do not cover. + var factory = NewInMemoryDbFactory(); + SeedCluster(factory, "C1", ("host-a:4053", "host-a", 4053, true, false)); + var psk = new RecordingPskProvider(); + + Sys.ActorOf(CentralCommunicationActor.Props( + factory, + ClusterClientMode(), + new RecordingClientFactory(() => CreateTestProbe()), + () => null, + dialer: null, + pskProvider: psk)); + + ExpectNoMsg(TimeSpan.FromMilliseconds(500)); + psk.KnownClusterPublications.ShouldBeEmpty(); + psk.InvalidateAllCount.ShouldBe(0); + } + + [Fact] + public void Grpc_mode_with_no_dialer_wired_drops_the_command_with_an_error() + { + // Mis-wiring, not misconfiguration: Mode=Grpc validates fine but the DI graph did not supply + // a dialer. Silence here would be a fleet-wide command outage with no symptom until a deploy + // stalled at its apply deadline two minutes later. + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false)); + var actor = Sys.ActorOf(CentralCommunicationActor.Props( + factory, + GrpcMode(), + new RecordingClientFactory(() => CreateTestProbe()), + () => null, + dialer: null, + pskProvider: null)); + + // Filtered on the DISPATCH message's exact phrase: PreStart logs its own error about the same + // mis-wiring, and a looser filter races it into a second match. + EventFilter.Error(contains: "no command dialer is wired").ExpectOne( + TimeSpan.FromSeconds(5), + () => actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch()))); + } + + private static DispatchDeployment NewDispatch() => new( + new DeploymentId(Guid.NewGuid()), new RevisionHash("abc"), new CorrelationId(Guid.NewGuid())); + + /// A spawned gRPC-mode actor plus the two seams the tests observe it through. + private sealed record GrpcRig(IActorRef Actor, RecordingDialer Dialer, RecordingPskProvider Psk); + + private GrpcRig SpawnGrpcRig( + IDbContextFactory factory, + RecordingDialer? dialer = null, + IMeshClusterClientFactory? clientFactory = null, + IActorRef? coordinator = null) + { + dialer ??= new RecordingDialer(); + var psk = new RecordingPskProvider(); + var actor = Sys.ActorOf(CentralCommunicationActor.Props( + factory, + GrpcMode(), + clientFactory ?? new RecordingClientFactory(() => CreateTestProbe()), + () => coordinator, + dialer, + psk)); + return new GrpcRig(actor, dialer, psk); + } + + /// + /// Waits until the actor has applied a contact load. The known-cluster publication is the + /// signal because it happens in the same handler, immediately after the target set is + /// adopted — and unlike the target set itself it is observable from outside the actor. A + /// command told before this point would be dropped as "no targets" and never retried, which + /// is correct behaviour and a flaky test. + /// + private void AwaitContactsLoaded(GrpcRig rig) => + AwaitCondition( + () => rig.Psk.KnownClusterPublications.Count >= 1, TimeSpan.FromSeconds(5)); + + /// Records every dial and answers however the test asks; never opens a socket. + private sealed class RecordingDialer : IMeshCommandDialer + { + private readonly object _gate = new(); + private readonly List _calls = []; + private readonly ManualResetEventSlim _released = new(false); + + /// + /// When true, every dial blocks its calling thread synchronously until + /// . Models the dialer's pre-await PSK lookup. + /// + public bool BlockBeforeFirstAwait { get; set; } + + /// Unblocks every dial parked by . + public void Release() => _released.Set(); + + /// Every dial made. Copied under the lock — the fan runs on thread-pool threads. + public IReadOnlyList Calls + { + get { lock (_gate) return _calls.ToList(); } + } + + /// Endpoints whose substring match makes the dial fault. + public string? FailEndpointsContaining { get; set; } + + /// Cluster ids whose dials fault. + public string[] FailClusters { get; set; } = []; + + /// + /// The fault raised for a matching dial. Fully qualified because this file's bare + /// Status is Akka's — an existing fact drives the actor's Status.Failure + /// handler, and importing Grpc.Core here would make that reference ambiguous. + /// + public Grpc.Core.RpcException? Fault { get; set; } + + public Task DispatchAsync( + string nodeEndpoint, CommandEnvelope envelope, string clusterId, CancellationToken ct) + { + lock (_gate) _calls.Add(new Dial(nodeEndpoint, envelope, clusterId)); + + if (BlockBeforeFirstAwait) + { + // Blocks the CALLING thread, exactly as IMeshPskProvider.TryGetPsk does on a cold + // cache: an async method runs synchronously to its first await, so a dial invoked + // straight from the actor would park the mailbox here. + _released.Wait(TimeSpan.FromSeconds(30)); + } + + var shouldFail = + (FailEndpointsContaining is not null && nodeEndpoint.Contains(FailEndpointsContaining)) + || FailClusters.Contains(clusterId, StringComparer.OrdinalIgnoreCase); + + return shouldFail && Fault is not null + ? Task.FromException(Fault) + : Task.CompletedTask; + } + + /// One recorded dial. + public sealed record Dial(string Endpoint, CommandEnvelope Envelope, string ClusterId); + } + + /// Records the gate and invalidation calls the contact refresh is supposed to make. + private sealed class RecordingPskProvider : IMeshPskProvider + { + private readonly object _gate = new(); + private readonly List> _publications = []; + private int _invalidateAllCount; + + /// Every known-cluster set published, in order. + public IReadOnlyList> KnownClusterPublications + { + get { lock (_gate) return _publications.ToList(); } + } + + /// How many times the whole cache was dropped. + public int InvalidateAllCount => Volatile.Read(ref _invalidateAllCount); + + public bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk) + { + psk = "psk"; + return true; + } + + public bool Matches(string clusterId, string presented) => true; + + public void SetKnownClusters(IReadOnlyCollection clusterIds) + { + lock (_gate) _publications.Add(clusterIds.ToList()); + } + + public void Invalidate(string clusterId) { } + + public void InvalidateAll() => Interlocked.Increment(ref _invalidateAllCount); + } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs new file mode 100644 index 00000000..96a0db0f --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs @@ -0,0 +1,271 @@ +using System.Diagnostics.CodeAnalysis; +using Grpc.Core; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication; + +/// +/// ClusterClient→gRPC Phase 1A — central's outbound command dial. One node, one call, no retry. +/// +/// +/// The facts that matter are the credential ones: the bearer must be the TARGET node's cluster's +/// key (not central's, and not some other cluster's), and a cluster with no resolvable key must +/// never be dialled at all. Everything else is plumbing. +/// +public sealed class MeshGrpcCommandDialerTests +{ + private const string Endpoint = "http://site-a-1:4056"; + + private static CommandEnvelope Envelope() => new() + { + CorrelationId = Guid.NewGuid().ToString("N"), + RestartDriver = new RestartDriverCommand + { + ClusterId = "SITE-A", + DriverInstanceId = "drv-1", + ActorByUserName = "operator", + CorrelationId = Guid.NewGuid().ToString("N"), + }, + }; + + private static MeshGrpcCommandDialer CreateDialer( + IMeshPskProvider psk, + RecordingClient client, + int dispatchTimeoutSeconds = 10) + => new( + psk, + Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = dispatchTimeoutSeconds }), + _ => client); + + [Fact] + public async Task Dial_presents_the_target_clusters_key_and_names_that_cluster() + { + // THE assertion. Central holds N keys; sending the wrong one turns a legitimate command into + // a PermissionDenied, and sending a fleet-wide one would be the blast radius this design + // exists to avoid. + var client = new RecordingClient(); + var dialer = CreateDialer( + new FakePskProvider(("SITE-A", "psk-site-a"), ("SITE-B", "psk-site-b")), client); + + await dialer.DispatchAsync(Endpoint, Envelope(), "SITE-B", CancellationToken.None); + + client.LastHeaders.ShouldNotBeNull(); + client.LastHeaders!.GetValue("authorization").ShouldBe("Bearer psk-site-b"); + client.LastHeaders.GetValue("x-otopcua-cluster").ShouldBe("SITE-B"); + } + + [Fact] + public async Task A_cluster_with_no_key_is_refused_locally_and_never_dialled() + { + // Fail-closed: not "dial and let the node reject it". An unauthenticated command attempt + // against a node's control surface should not exist even as a rejected one. + var client = new RecordingClient(); + var dialer = CreateDialer(new FakePskProvider(("SITE-A", "psk-site-a")), client); + + var ex = await Should.ThrowAsync( + () => dialer.DispatchAsync(Endpoint, Envelope(), "SITE-Z", CancellationToken.None)); + + ex.StatusCode.ShouldBe(StatusCode.Unauthenticated); + client.Calls.ShouldBe(0); + } + + [Fact] + public async Task The_envelope_reaches_the_node_unchanged() + { + var client = new RecordingClient(); + var dialer = CreateDialer(new FakePskProvider(("SITE-A", "psk-site-a")), client); + var envelope = Envelope(); + + await dialer.DispatchAsync(Endpoint, envelope, "SITE-A", CancellationToken.None); + + client.LastRequest.ShouldBe(envelope); + } + + [Fact] + public async Task A_node_rejection_surfaces_as_its_own_RpcException() + { + // The actor grades per-node outcomes off the status code, so it must arrive intact rather + // than wrapped in an AggregateException or swallowed here. + var client = new RecordingClient + { + Fault = new RpcException(new Status(StatusCode.PermissionDenied, "nope")), + }; + var dialer = CreateDialer(new FakePskProvider(("SITE-A", "psk-site-a")), client); + + var ex = await Should.ThrowAsync( + () => dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", CancellationToken.None)); + + ex.StatusCode.ShouldBe(StatusCode.PermissionDenied); + } + + [Fact] + public async Task The_per_call_deadline_bounds_a_node_that_never_answers() + { + // A dispatch dials every node in the fleet. Without this bound one unreachable node holds a + // task and a socket for as long as the process lives, while the deploy path's own apply + // deadline expires around it. + var client = new RecordingClient { HangUntilCancelled = true }; + var dialer = CreateDialer( + new FakePskProvider(("SITE-A", "psk-site-a")), client, dispatchTimeoutSeconds: 1); + + await Should.ThrowAsync( + () => dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", CancellationToken.None) + .WaitAsync(TimeSpan.FromSeconds(10))); + } + + [Fact] + public async Task The_callers_token_still_cancels_a_dial_inside_the_deadline() + { + // Linked, not replaced: actor shutdown must cancel everything in flight even when the + // per-call deadline is nowhere near expiring. + var client = new RecordingClient { HangUntilCancelled = true }; + var dialer = CreateDialer( + new FakePskProvider(("SITE-A", "psk-site-a")), client, dispatchTimeoutSeconds: 600); + using var cts = new CancellationTokenSource(); + + var dial = dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", cts.Token); + await cts.CancelAsync(); + + await Should.ThrowAsync(() => dial.WaitAsync(TimeSpan.FromSeconds(10))); + } + + [Fact] + public async Task A_zero_dispatch_timeout_leaves_the_dial_bounded_only_by_the_callers_token() + { + // The validator requires > 0 under Mode=Grpc, so this is the misconfigured/other-mode shape. + // It must not be read as "expire immediately" — that would drop every command instantly. + var client = new RecordingClient(); + var dialer = CreateDialer( + new FakePskProvider(("SITE-A", "psk-site-a")), client, dispatchTimeoutSeconds: 0); + + await dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", CancellationToken.None); + + client.Calls.ShouldBe(1); + } + + [Fact] + public async Task Dialing_after_disposal_throws_rather_than_repopulating_the_channel_cache() + { + var client = new RecordingClient(); + var dialer = CreateDialer(new FakePskProvider(("SITE-A", "psk-site-a")), client); + dialer.Dispose(); + + await Should.ThrowAsync( + () => dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", CancellationToken.None)); + } + + [Fact] + public async Task The_real_channel_factory_dials_and_is_disposable() + { + // The fake client seam covers every other fact here, which means nothing else ever exercises + // GrpcChannel.ForAddress or the cache it fills. Dial an endpoint nothing is listening on: + // the failure must be an ordinary RpcException the actor can log, not a construction throw, + // and Dispose must then clean up the channel the dial created. + var dialer = new MeshGrpcCommandDialer( + new FakePskProvider(("SITE-A", "psk-site-a")), + Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 5 })); + + await Should.ThrowAsync( + () => dialer.DispatchAsync( + "http://127.0.0.1:1", Envelope(), "SITE-A", CancellationToken.None)); + + Should.NotThrow(dialer.Dispose); + } + + [Fact] + public void It_activates_from_the_container_despite_the_optional_client_factory_parameter() + { + // The seam is an optional constructor parameter, which MS.DI honours only because it checks + // HasDefaultValue. Get that wrong — by making it non-optional, or by adding a second public + // constructor — and the failure is a startup crash on the admin node, invisible to every + // other test here because they all construct the dialer by hand. + var services = new ServiceCollection(); + services.AddSingleton(new FakePskProvider(("SITE-A", "psk-site-a"))); + services.AddSingleton>( + Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 10 })); + services.AddSingleton(); + + using var provider = services.BuildServiceProvider(); + + provider.GetRequiredService().ShouldBeOfType(); + } + + /// + /// A that records what it was + /// handed and answers however the test asks. The generated client's protected parameterless + /// constructor exists precisely for this. + /// + private sealed class RecordingClient : NodeCommandService.NodeCommandServiceClient + { + /// Number of calls that reached the client. + public int Calls { get; private set; } + + /// Headers of the most recent call. + public Metadata? LastHeaders { get; private set; } + + /// Request of the most recent call. + public CommandEnvelope? LastRequest { get; private set; } + + /// When set, the call faults with this instead of answering. + public RpcException? Fault { get; set; } + + /// When true, the call never completes until its token is cancelled. + public bool HangUntilCancelled { get; set; } + + public override AsyncUnaryCall ExecuteAsync( + CommandEnvelope request, + Metadata? headers = null, + DateTime? deadline = null, + CancellationToken cancellationToken = default) + { + Calls++; + LastHeaders = headers; + LastRequest = request; + + var response = Fault is not null + ? Task.FromException(Fault) + : HangUntilCancelled + ? HangAsync(cancellationToken) + : Task.FromResult(new ExecuteAck()); + + return new AsyncUnaryCall( + response, + Task.FromResult(new Metadata()), + () => Status.DefaultSuccess, + () => [], + () => { }); + } + + private static async Task HangAsync(CancellationToken ct) + { + await Task.Delay(Timeout.InfiniteTimeSpan, ct).ConfigureAwait(false); + return new ExecuteAck(); + } + } + + /// An over a fixed key set; unknown clusters fail closed. + private sealed class FakePskProvider(params (string Cluster, string Psk)[] keys) : IMeshPskProvider + { + private readonly Dictionary _keys = + keys.ToDictionary(k => k.Cluster, k => k.Psk, StringComparer.OrdinalIgnoreCase); + + public bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk) + => _keys.TryGetValue(clusterId, out psk); + + public bool Matches(string clusterId, string presented) + => _keys.TryGetValue(clusterId, out var expected) + && string.Equals(expected, presented, StringComparison.Ordinal); + + public void SetKnownClusters(IReadOnlyCollection clusterIds) { } + + public void Invalidate(string clusterId) { } + + public void InvalidateAll() { } + } +} From 48dc2120c659867b5e49164ca1b94d7dc8e1ee4d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 6 Aug 2026 06:44:20 -0400 Subject: [PATCH 05/12] fix(mesh): gate follows the endpoint not the mode + review fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The known-cluster gate published only under Mode=Grpc, but the interceptor it feeds is registered on every admin node and 1B maps CentralCommandService by LISTENER PORT — neither consults the mode. A dark-switch rollback to ClusterClient or Dps would therefore have left that endpoint serving with the gate permanently inactive, silently restoring both the timing oracle and the pre-auth secret-store/audit flood, as a side effect of a rollback nobody would connect to authentication. The gate is central's cluster inventory, not transport state, so it is now decoupled from the mode entirely: - SetKnownClusters + InvalidateAll run on EVERY successful contact load in EVERY mode, and the contact-refresh timer starts in all three. Under Dps this is a ConfigDb query every ContactRefreshSeconds that did not run before; central always has a ConfigDb, and one indexed read a minute is the cheaper half of the trade. - The GATE set is built from ALL ClusterNode rows. The eligibility filter moved out of the query's WHERE and into the loop, so the FAN set is byte-for-byte what it was (enabled && !maintenance) while the gate widens. This also dissolves the maintenance coupling flagged in 1A.2: a cluster whose nodes are all disabled or in maintenance can still ack, and refusing it would make maintenance mode an authentication switch. Central still refuses clusters it has never heard of. - HandleContactsLoaded keeps a documented order — targets adopted, gate published, cache dropped — now pinned by test. Guards: PreStart mirrors the null-dialer error for a null PSK provider (silent absence is the worst outcome there — nothing malfunctions, so nothing is investigated, while the gate never publishes and keys never rotate); a failed contact load now says out loud that the gate is stale and, before any load has succeeded, inactive; MeshGrpcCommandDialer clamps a non-positive DispatchTimeoutSeconds to 10 with a construction Warning, because the validator for that key is driver-role-scoped and a pure-admin central reads it unvalidated; the dialer's local PSK refusal was shortened so its remediation survives the actor's 200-char truncation of remote-authored status detail; the portless-node warn-once set is rebuilt each refresh, so a node that regains then loses a port warns again — the regression is the occurrence worth seeing. Docs/pins: dialer disposal comment no longer claims one-way; the two interceptor-test comments claiming Matches has a default implementation corrected (it deliberately has none); new pins for per-refresh invalidation, the SetKnownClusters-before-InvalidateAll order, and one channel per endpoint via a new CachedChannelCount. The_known_cluster_set_is_not_published_by_the_other_two_modes is replaced — its premise is obsolete — by three facts: every mode publishes, gate set is a strict superset of the fan set, and the handler ordering. ControlPlane.Tests 230 -> 236, Host.Tests 92, Cluster.Tests 184, 0 warnings. --- .../CentralCommunicationActor.cs | 162 +++++++++++++----- .../Communication/MeshGrpcCommandDialer.cs | 61 +++++-- .../CentralCommunicationActorTests.cs | 134 +++++++++++++-- .../MeshGrpcCommandDialerTests.cs | 85 ++++++++- .../CentralCommandAuthInterceptorTests.cs | 10 +- 5 files changed, 368 insertions(+), 84 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs index 635194ff..60f3b7c2 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs @@ -159,13 +159,23 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers // an operator cannot distinguish "no nodes configured" from "the database is down", and the // contact set silently freezes at whatever it last held. // - // Under Grpc that freeze extends to the known-cluster gate, deliberately: a failed load must - // neither widen the gate nor narrow it, and the previous set is the last thing central - // actually knew. - Receive(f => _log.Warning( - f.Cause, - "Failed to load ClusterNode contact points; the mesh contact set was NOT refreshed and " - + "may be stale or empty. Commands may be dropped until the next refresh")); + // The freeze extends to the known-cluster gate, deliberately: a failed load must neither + // widen the gate nor narrow it, and the previous set is the last thing central actually knew. + // The second line exists because that freeze is INVISIBLE otherwise, and it is fail-OPEN + // before the first successful load — a central that has never reached its ConfigDb refuses + // no cluster at all. + Receive(f => + { + _log.Warning( + f.Cause, + "Failed to load ClusterNode contact points; the mesh contact set was NOT refreshed " + + "and may be stale or empty. Commands may be dropped until the next refresh"); + _log.Warning( + "The known-cluster gate was NOT refreshed either: it still holds the set from the " + + "last successful load, and if none has succeeded yet (boot, or central SQL Server " + + "unreachable since start) it is INACTIVE — every cluster id reaches the secret " + + "store, which is the pre-authentication cost the gate exists to remove"); + }); Receive(_ => { /* DPS subscribe confirmation */ }); } @@ -177,8 +187,9 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers { _log.Info( "Mesh transport is {Mode}: commands are published on DistributedPubSub and no " - + "ClusterClient is created", _options.Mode); - return; + + "ClusterClient is created. The contact refresh still runs — it is what keeps the " + + "known-cluster gate current, and that gate belongs to the inbound ack endpoint, " + + "which is mapped by port and does not care about this mode", _options.Mode); } if (_useGrpc && _dialer is null) @@ -192,8 +203,25 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers + "IMeshCommandDialer"); } - // The refresh drives more than ClusterClient contacts under Grpc: it is also what publishes - // the dispatch target set and keeps the PSK provider's known-cluster gate current. + if (_pskProvider is null) + { + // Mode-independent, like the gate itself. A silent absence here is the worst of the + // three outcomes in this method: nothing malfunctions, so nothing is ever investigated, + // while the gate stays permanently inactive and a rotated key is never picked up. + _log.Error( + "No IMeshPskProvider was supplied to this actor; the known-cluster gate will NEVER " + + "be published and cached per-cluster keys will NEVER be dropped. Inbound mesh acks " + + "stay exposed to the pre-authentication secret-store cost the gate removes, and a " + + "rotated key needs a central restart. Check the ControlPlane registration"); + } + + // Started in EVERY mode. The refresh is not transport state: it is central's ClusterNode + // inventory, and the known-cluster gate it feeds protects an endpoint mapped by PORT, not by + // MeshTransport:Mode. Gating the timer on the mode would mean a dark-switch rollback to + // ClusterClient or Dps left that endpoint live with the gate permanently inactive — silently + // reopening both the timing oracle and the pre-auth store/audit flood. Under Dps this is a + // ConfigDb query every ContactRefreshSeconds that did not run before; central always has a + // ConfigDb, and one indexed read a minute is the cheaper half of the trade. Timers.StartPeriodicTimer( RefreshTimerKey, new RefreshContacts(), @@ -487,10 +515,18 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers Task.Run(async () => { await using var db = await dbFactory.CreateDbContextAsync(ct).ConfigureAwait(false); + // UNFILTERED, with the eligibility flags projected and applied per row below. The two + // outputs of this read want different row sets: the contact/dispatch sets are the + // enabled, non-maintenance nodes central talks TO, while the known-cluster gate answers + // "is this a cluster central has ever heard of" and must include a cluster whose every + // node is disabled or in maintenance — such a cluster can still send an ack, and denying + // it would make maintenance mode an authentication switch. var rows = await db.ClusterNodes .AsNoTracking() - .Where(n => n.Enabled && !n.MaintenanceMode) - .Select(n => new { n.NodeId, n.ClusterId, n.Host, n.AkkaPort, n.GrpcPort }) + .Select(n => new + { + n.NodeId, n.ClusterId, n.Host, n.AkkaPort, n.GrpcPort, n.Enabled, n.MaintenanceMode, + }) .ToListAsync(ct) .ConfigureAwait(false); @@ -506,15 +542,21 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers var targets = new List(rows.Count); var portless = new List(); - // Every cluster with at least one enabled, non-maintenance row — the known-cluster gate's - // membership set. Deliberately independent of GrpcPort: the gate answers "is this a cluster - // central knows", and a cluster central cannot currently dial can still send it an ack. + // Every cluster with ANY ClusterNode row — the known-cluster gate's membership set, and + // deliberately a superset of the fan set. Independent of Enabled, MaintenanceMode and + // GrpcPort alike: none of those describe whether the cluster exists. var clusterIds = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (var row in rows) { clusterIds.Add(row.ClusterId); + // The eligibility filter that used to be the query's WHERE clause. Everything below + // this line — receptionist contacts and gRPC targets — sees exactly the rows it saw + // before. + if (!row.Enabled || row.MaintenanceMode) + continue; + if (row.GrpcPort is { } grpcPort) targets.Add(new MeshNodeTarget( row.NodeId, row.ClusterId, $"http://{row.Host}:{grpcPort}")); @@ -548,11 +590,16 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers private void HandleContactsLoaded(ContactsLoaded msg) { + // Order is load-bearing and is pinned by test: the dispatch set is adopted BEFORE the gate is + // published, so anything that observes the gate can rely on the targets already being current. if (_useGrpc) - { - HandleGrpcContactsLoaded(msg); - return; - } + AdoptGrpcTargets(msg); + + // EVERY mode. See PreStart — the gate protects an endpoint mapped by port, not by mode. + PublishKnownClusters(msg); + + if (!_useClusterClient) + return; // Dps or Grpc: no ClusterClient to rebuild. foreach (var bad in msg.Malformed) { @@ -613,33 +660,28 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers } /// - /// Adopts a contact load under : the dispatch - /// target set, and the two things the PSK provider needs kept current. + /// Adopts a contact load's gRPC dispatch set. only + /// — the other two transports have no use for it. /// /// The load result. /// - /// - /// The malformed-receptionist-address warnings are deliberately NOT logged here: nothing - /// on this transport dials a receptionist, and reporting an unusable Akka address as a - /// command-plane problem would send an operator after the wrong column. A host that is - /// unusable as a gRPC endpoint surfaces instead as a named per-node dial failure on every - /// dispatch, which is a truer signal. - /// - /// - /// InvalidateAll, not per-cluster Invalidate. The requirement is that - /// a rotated key takes effect within one refresh, and only a whole-cache drop delivers - /// that: a key can rotate without the cluster's membership changing at all, so invalidating - /// only departed clusters would serve the pre-rotation key forever. The cost is one store - /// read per cluster per refresh — the cache holds a single entry per cluster — and it is - /// paid only by clusters that are actually dialled. - /// + /// The malformed-receptionist-address warnings are deliberately NOT logged here: nothing on + /// this transport dials a receptionist, and reporting an unusable Akka address as a + /// command-plane problem would send an operator after the wrong column. A host that is + /// unusable as a gRPC endpoint surfaces instead as a named per-node dial failure on every + /// dispatch, which is a truer signal. /// - private void HandleGrpcContactsLoaded(ContactsLoaded msg) + private void AdoptGrpcTargets(ContactsLoaded msg) { + // Rebuilt against this refresh's skip list, not merely added to. A node that gained a port + // and later lost it again must warn a second time — the operator's mental model is "the last + // refresh told me", not "you were told once, in a log that has since rotated away". + _warnedPortlessNodes.IntersectWith(msg.PortlessNodes); + foreach (var nodeId in msg.PortlessNodes) { - // Warn-once per node id: a persistently portless row is re-skipped every refresh, and the - // same repetition problem TelemetryNodeSource solved this way. + // Warn-once per node id per continuous skip run: a persistently portless row is + // re-skipped every refresh, and that is the repetition TelemetryNodeSource suppresses. if (_warnedPortlessNodes.Add(nodeId)) { _log.Warning( @@ -651,11 +693,6 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers _targets = msg.Targets; - // Published AFTER the target set is adopted, so anything observing the gate can rely on the - // dispatch set already being current. - _pskProvider?.SetKnownClusters(msg.ClusterIds); - _pskProvider?.InvalidateAll(); - if (_targets.Count == 0) { _log.Warning( @@ -665,6 +702,35 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers } } + /// + /// Hands the PSK provider this refresh's cluster inventory and drops its key cache. Runs in + /// every transport mode. + /// + /// The load result. + /// + /// + /// Not gated on , on purpose. The gate feeds + /// CentralCommandAuthInterceptor, which is registered on every admin node, and the + /// ack service it protects is mapped by listener port — neither consults the mode. Gating + /// this on Grpc would mean a dark-switch rollback to ClusterClient or + /// Dps left that endpoint serving with a permanently inactive gate: the timing + /// oracle and the pre-authentication store/audit flood would both come back, silently, + /// as a side effect of a rollback nobody would connect to authentication. + /// + /// + /// InvalidateAll, not per-cluster Invalidate. The requirement is that + /// a rotated key takes effect within one refresh, and only a whole-cache drop delivers + /// that: a key can rotate without the cluster's membership changing at all, so + /// invalidating only departed clusters would serve the pre-rotation key forever. The cost + /// is one store read per cluster per refresh — the cache holds a single entry per cluster. + /// + /// + private void PublishKnownClusters(ContactsLoaded msg) + { + _pskProvider?.SetKnownClusters(msg.ClusterIds); + _pskProvider?.InvalidateAll(); + } + /// Stops a cluster's existing client and creates a replacement for its new contact set. /// /// @@ -740,8 +806,10 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers /// the load task so the warn-once bookkeeping stays on the actor's thread. /// /// - /// Every application Cluster with at least one enabled, non-maintenance row — - /// the PSK provider's known-cluster gate membership. + /// Every application Cluster with any ClusterNode row — the PSK + /// provider's known-cluster gate membership, and deliberately a superset of + /// : a cluster whose nodes are all disabled or in maintenance can + /// still send an ack, and refusing it would make maintenance mode an authentication switch. /// public sealed record ContactsLoaded( IReadOnlyDictionary> ContactsByCluster, diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs index 46dcd830..10d9761f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs @@ -1,6 +1,7 @@ using System.Collections.Concurrent; using Grpc.Core; using Grpc.Net.Client; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1; @@ -63,13 +64,21 @@ public interface IMeshCommandDialer /// caller's token so actor shutdown still cancels everything in flight. /// /// -/// Registered as a singleton and disposed by the container. Disposal is one-way: a dial -/// after it throws rather than silently repopulating the cache with channels nobody will -/// ever dispose. +/// Registered as a singleton and disposed by the container. A dial started after disposal +/// throws, so no new work repopulates the cache — but a dial already past that check can +/// still add a channel after the cache was cleared. That leak is bounded to the dials in +/// flight at shutdown, which is the only time disposal happens, so it is accepted rather +/// than serialized against. /// /// public sealed class MeshGrpcCommandDialer : IMeshCommandDialer, IDisposable { + /// + /// Substituted for a non-positive . + /// Deliberately the option's own default rather than a second value to reconcile. + /// + private const int FallbackDispatchTimeoutSeconds = 10; + /// /// Header naming the cluster the presented bearer belongs to. Central's own inbound gate /// reads the same header on the ack path; sending it outbound keeps one convention and tells @@ -78,6 +87,7 @@ public sealed class MeshGrpcCommandDialer : IMeshCommandDialer, IDisposable private const string ClusterHeader = "x-otopcua-cluster"; private readonly IMeshPskProvider _pskProvider; + private readonly ILogger _log; private readonly int _dispatchTimeoutSeconds; private readonly Func _clientFactory; private readonly ConcurrentDictionary _channels = new(StringComparer.Ordinal); @@ -87,23 +97,53 @@ public sealed class MeshGrpcCommandDialer : IMeshCommandDialer, IDisposable /// Initializes a new instance of the class. /// Supplies the bearer for the target node's cluster. /// Mesh transport options; supplies the per-call deadline. + /// Logger for the construction-time deadline check. /// /// Seam for tests: builds the generated client for an endpoint. Defaults to a real h2c /// , one cached per endpoint. /// + /// + /// A non-positive DispatchTimeoutSeconds is clamped, not honoured. Zero would mean an + /// unbounded call — the exact failure the deadline exists to prevent — and the validator that + /// rejects it is scoped to driver-role nodes, so the one deployment shape that reads this + /// value unvalidated is a pure-admin central. Clamping loudly is better than a fleet-wide fan + /// that can never time out. + /// public MeshGrpcCommandDialer( IMeshPskProvider pskProvider, IOptions options, + ILogger log, Func? clientFactory = null) { ArgumentNullException.ThrowIfNull(pskProvider); ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(log); _pskProvider = pskProvider; - _dispatchTimeoutSeconds = options.Value.DispatchTimeoutSeconds; + _log = log; + + var configured = options.Value.DispatchTimeoutSeconds; + if (configured <= 0) + { + _log.LogWarning( + "MeshTransport:DispatchTimeoutSeconds is {Configured}, which would leave every mesh " + + "command dial unbounded; using {Fallback}s instead. Set a positive value — the " + + "boot validator for this key covers driver-role nodes only, so a pure-admin central " + + "reaches this line rather than failing at startup.", + configured, FallbackDispatchTimeoutSeconds); + } + + _dispatchTimeoutSeconds = configured > 0 ? configured : FallbackDispatchTimeoutSeconds; _clientFactory = clientFactory ?? DefaultClientFactory; } + /// + /// Gets the number of endpoints holding a cached . Exposed so the + /// cache — the whole defence against a socket per call across a fleet-wide fan — is + /// assertable rather than merely intended. + /// + public int CachedChannelCount => _channels.Count; + /// public async Task DispatchAsync( string nodeEndpoint, CommandEnvelope envelope, string clusterId, CancellationToken ct) @@ -117,12 +157,14 @@ public sealed class MeshGrpcCommandDialer : IMeshCommandDialer, IDisposable { // The cluster id is operator-authored configuration, not a secret, so naming it is what // makes this actionable. The key never appears here or anywhere else in this class. + // Kept short on purpose: the actor truncates a status detail at 200 characters because a + // REMOTE node authors it, and this locally-authored one has to survive that cap with its + // remediation intact. It measures ~160 characters with a six-character cluster id, so an + // id would have to exceed ~40 characters to lose the trailing rig hint. throw new RpcException(new Status( StatusCode.Unauthenticated, - $"No mesh pre-shared key could be established for cluster '{clusterId}'; the dial was " - + "refused locally rather than made unauthenticated. Seed the secret " - + "OT-MESH-PSK- on this node, or set MeshTransport:ClusterPsks: " - + "on a rig.")); + $"No mesh pre-shared key for cluster '{clusterId}'; dial refused locally. Seed secret " + + "OT-MESH-PSK-, or set MeshTransport:ClusterPsks: on a rig.")); } var headers = new Metadata @@ -132,8 +174,7 @@ public sealed class MeshGrpcCommandDialer : IMeshCommandDialer, IDisposable }; using var deadlineCts = CancellationTokenSource.CreateLinkedTokenSource(ct); - if (_dispatchTimeoutSeconds > 0) - deadlineCts.CancelAfter(TimeSpan.FromSeconds(_dispatchTimeoutSeconds)); + deadlineCts.CancelAfter(TimeSpan.FromSeconds(_dispatchTimeoutSeconds)); var client = _clientFactory(nodeEndpoint); using var call = client.ExecuteAsync(envelope, headers, cancellationToken: deadlineCts.Token); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs index 8563f781..a3df8f20 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs @@ -461,6 +461,28 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase rig.Dialer.Calls[0].Endpoint.ShouldBe("http://host-b:4056"); } + [Fact] + public void A_node_that_regains_then_loses_its_GrpcPort_warns_again() + { + // Warn-once is per continuous skip run, not per process. The suppression exists so a + // permanently portless row does not repeat every minute; carried across a repair it would + // silence the SECOND time a node loses its endpoint — the occurrence an operator most needs + // to see, because it is a regression rather than a known gap. + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", null, true, false)); + var rig = SpawnGrpcRig(factory); + AwaitContactsLoaded(rig); + + SetGrpcPort(factory, "host-a:4053", 4056); + rig.Actor.Tell(new CentralCommunicationActor.RefreshContacts()); + AwaitCondition( + () => rig.Psk.KnownClusterPublications.Count >= 2, TimeSpan.FromSeconds(5)); + + SetGrpcPort(factory, "host-a:4053", null); + EventFilter.Warning(contains: "has no GrpcPort").ExpectOne(TimeSpan.FromSeconds(5), () => + rig.Actor.Tell(new CentralCommunicationActor.RefreshContacts())); + } + [Fact] public void One_failing_node_does_not_stop_the_fan_and_is_logged_by_node_id() { @@ -690,29 +712,89 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5)); rig.Dialer.Calls.Select(c => c.Endpoint).ShouldBe( new[] { "http://host-a:4056", "http://host-b:4056" }, ignoreOrder: true); + + // Once PER REFRESH, not once at startup. A key can rotate without any membership change, so + // an invalidation that only fired when the cluster set moved would serve the pre-rotation + // key forever and a rotation would need a central restart. + rig.Psk.InvalidateAllCount.ShouldBeGreaterThanOrEqualTo(2); } - [Fact] - public void The_known_cluster_set_is_not_published_by_the_other_two_modes() + [Theory] + [InlineData(MeshTransportOptions.ModeDps)] + [InlineData(MeshTransportOptions.ModeClusterClient)] + [InlineData(MeshTransportOptions.ModeGrpc)] + public void The_known_cluster_gate_is_published_in_every_transport_mode(string mode) { - // The gate is a gRPC-command-plane control. Publishing it under Dps or ClusterClient would - // start refusing acks on a transport whose ack path never consults the provider at all — - // and, worse, would do so from a code path those modes' tests do not cover. + // Replaces an earlier fact asserting the OPPOSITE, whose premise did not survive 1B: the + // gate feeds CentralCommandAuthInterceptor, which is registered on every admin node, and the + // ack service it protects is mapped by LISTENER PORT — neither consults MeshTransport:Mode. + // Gating the publication on Grpc meant a dark-switch rollback to ClusterClient or Dps left + // that endpoint serving with a permanently inactive gate, silently restoring both the timing + // oracle and the pre-authentication store/audit flood. The set is central's cluster + // inventory, not transport state. var factory = NewInMemoryDbFactory(); - SeedCluster(factory, "C1", ("host-a:4053", "host-a", 4053, true, false)); + SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false)); var psk = new RecordingPskProvider(); Sys.ActorOf(CentralCommunicationActor.Props( factory, - ClusterClientMode(), + new MeshTransportOptions { Mode = mode, ContactRefreshSeconds = 60 }, new RecordingClientFactory(() => CreateTestProbe()), () => null, - dialer: null, - pskProvider: psk)); + new RecordingDialer(), + psk)); - ExpectNoMsg(TimeSpan.FromMilliseconds(500)); - psk.KnownClusterPublications.ShouldBeEmpty(); - psk.InvalidateAllCount.ShouldBe(0); + AwaitCondition( + () => psk.KnownClusterPublications.Count >= 1, TimeSpan.FromSeconds(5)); + psk.KnownClusterPublications[0].ShouldBe(new[] { "C1" }); + psk.InvalidateAllCount.ShouldBeGreaterThanOrEqualTo(1); + } + + [Fact] + public void The_gate_set_includes_a_cluster_the_fan_set_excludes() + { + // The two sets answer different questions. The FAN asks "who does central talk to" — enabled, + // non-maintenance, ported. The GATE asks "is this a cluster central has ever heard of", and + // must say yes for a cluster whose every node is disabled or in maintenance: such a cluster + // can still send a deployment ack, and refusing it would quietly turn maintenance mode into + // an authentication switch. + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false)); + SeedGrpcCluster(factory, "C-DISABLED", ("host-b:4053", "host-b", 4056, false, false)); + SeedGrpcCluster(factory, "C-MAINT", ("host-c:4053", "host-c", 4056, true, true)); + var rig = SpawnGrpcRig(factory); + AwaitContactsLoaded(rig); + + rig.Psk.KnownClusterPublications[0].ShouldBe( + new[] { "C1", "C-DISABLED", "C-MAINT" }, ignoreOrder: true); + + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch())); + + AwaitCondition(() => rig.Dialer.Calls.Count == 1, TimeSpan.FromSeconds(5)); + ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + rig.Dialer.Calls.Count.ShouldBe(1); + rig.Dialer.Calls[0].ClusterId.ShouldBe("C1"); + } + + [Fact] + public void The_gate_is_published_after_the_dispatch_set_is_adopted_and_before_the_cache_is_dropped() + { + // Two orderings in one handler, both load-bearing. SetKnownClusters must precede + // InvalidateAll, or the first lookup after a refresh re-reads the store under the PREVIOUS + // membership — briefly resolving a cluster central has just retired. And the dispatch set is + // adopted first so a command processed immediately after the publication already fans over + // the current targets. + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false)); + var rig = SpawnGrpcRig(factory); + AwaitContactsLoaded(rig); + + rig.Psk.Sequence.ShouldBe(new[] { "SetKnownClusters", "InvalidateAll" }); + + // Targets-before-gate, observed the only way it can be from outside the actor: the very next + // message finds the dispatch set already populated. + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch())); + AwaitCondition(() => rig.Dialer.Calls.Count == 1, TimeSpan.FromSeconds(5)); } [Fact] @@ -738,6 +820,15 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase () => actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch()))); } + private static void SetGrpcPort( + IDbContextFactory factory, string nodeId, int? grpcPort) + { + using var db = factory.CreateDbContext(); + var node = db.ClusterNodes.Single(n => n.NodeId == nodeId); + node.GrpcPort = grpcPort; + db.SaveChanges(); + } + private static DispatchDeployment NewDispatch() => new( new DeploymentId(Guid.NewGuid()), new RevisionHash("abc"), new CorrelationId(Guid.NewGuid())); @@ -839,8 +930,15 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase { private readonly object _gate = new(); private readonly List> _publications = []; + private readonly List _sequence = []; private int _invalidateAllCount; + /// Names of the mutating calls made, in the order they arrived. + public IReadOnlyList Sequence + { + get { lock (_gate) return _sequence.ToList(); } + } + /// Every known-cluster set published, in order. public IReadOnlyList> KnownClusterPublications { @@ -860,11 +958,19 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase public void SetKnownClusters(IReadOnlyCollection clusterIds) { - lock (_gate) _publications.Add(clusterIds.ToList()); + lock (_gate) + { + _publications.Add(clusterIds.ToList()); + _sequence.Add(nameof(SetKnownClusters)); + } } public void Invalidate(string clusterId) { } - public void InvalidateAll() => Interlocked.Increment(ref _invalidateAllCount); + public void InvalidateAll() + { + Interlocked.Increment(ref _invalidateAllCount); + lock (_gate) _sequence.Add(nameof(InvalidateAll)); + } } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs index 96a0db0f..be0896aa 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs @@ -1,6 +1,8 @@ using System.Diagnostics.CodeAnalysis; using Grpc.Core; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Shouldly; using Xunit; @@ -41,6 +43,7 @@ public sealed class MeshGrpcCommandDialerTests => new( psk, Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = dispatchTimeoutSeconds }), + NullLogger.Instance, _ => client); [Fact] @@ -136,17 +139,25 @@ public sealed class MeshGrpcCommandDialerTests } [Fact] - public async Task A_zero_dispatch_timeout_leaves_the_dial_bounded_only_by_the_callers_token() + public async Task A_non_positive_dispatch_timeout_is_clamped_rather_than_honoured() { - // The validator requires > 0 under Mode=Grpc, so this is the misconfigured/other-mode shape. - // It must not be read as "expire immediately" — that would drop every command instantly. - var client = new RecordingClient(); - var dialer = CreateDialer( - new FakePskProvider(("SITE-A", "psk-site-a")), client, dispatchTimeoutSeconds: 0); + // Zero would mean an UNBOUNDED call — the exact failure the deadline exists to prevent — and + // MeshTransportOptionsValidator's rule for this key is scoped to driver-role nodes, so a + // pure-admin central reaches this code with the value unvalidated. Clamp loudly. + var client = new RecordingClient { HangUntilCancelled = true }; + var logger = new RecordingLogger(); + var dialer = new MeshGrpcCommandDialer( + new FakePskProvider(("SITE-A", "psk-site-a")), + Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 0 }), + logger, + _ => client); - await dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", CancellationToken.None); + logger.Warnings.ShouldContain(w => w.Contains("DispatchTimeoutSeconds")); - client.Calls.ShouldBe(1); + // And the clamp is real, not just logged: the dial must still expire on its own. + await Should.ThrowAsync( + () => dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", CancellationToken.None) + .WaitAsync(TimeSpan.FromSeconds(20))); } [Fact] @@ -169,7 +180,8 @@ public sealed class MeshGrpcCommandDialerTests // and Dispose must then clean up the channel the dial created. var dialer = new MeshGrpcCommandDialer( new FakePskProvider(("SITE-A", "psk-site-a")), - Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 5 })); + Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 5 }), + NullLogger.Instance); await Should.ThrowAsync( () => dialer.DispatchAsync( @@ -178,6 +190,38 @@ public sealed class MeshGrpcCommandDialerTests Should.NotThrow(dialer.Dispose); } + [Fact] + public async Task Two_dials_to_one_endpoint_share_a_single_channel() + { + // The cache is the whole defence against a socket per call: a deploy dispatch fans across + // every node in the fleet, and it repeats on every redeploy. A channel per dial exhausts + // ports on central long before anything else notices. + var dialer = new MeshGrpcCommandDialer( + new FakePskProvider(("SITE-A", "psk-site-a")), + Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 5 }), + NullLogger.Instance); + + // Nothing is listening, so both dials fail — irrelevant here: the channel is constructed + // before the call is placed, which is exactly what is being counted. + for (var i = 0; i < 2; i++) + { + await Should.ThrowAsync( + () => dialer.DispatchAsync( + "http://127.0.0.1:1", Envelope(), "SITE-A", CancellationToken.None)); + } + + dialer.CachedChannelCount.ShouldBe(1); + + // A second endpoint is a second channel — the cache keys on the endpoint, it does not + // collapse every node onto one connection. + await Should.ThrowAsync( + () => dialer.DispatchAsync( + "http://127.0.0.1:2", Envelope(), "SITE-A", CancellationToken.None)); + + dialer.CachedChannelCount.ShouldBe(2); + Should.NotThrow(dialer.Dispose); + } + [Fact] public void It_activates_from_the_container_despite_the_optional_client_factory_parameter() { @@ -186,6 +230,7 @@ public sealed class MeshGrpcCommandDialerTests // constructor — and the failure is a startup crash on the admin node, invisible to every // other test here because they all construct the dialer by hand. var services = new ServiceCollection(); + services.AddLogging(); services.AddSingleton(new FakePskProvider(("SITE-A", "psk-site-a"))); services.AddSingleton>( Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 10 })); @@ -249,6 +294,28 @@ public sealed class MeshGrpcCommandDialerTests } } + /// Captures Warning-level messages so a construction-time check is assertable. + private sealed class RecordingLogger : ILogger + { + /// Every warning logged, formatted. + public List Warnings { get; } = []; + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + if (logLevel == LogLevel.Warning) + Warnings.Add(formatter(state, exception)); + } + } + /// An over a fixed key set; unknown clusters fail closed. private sealed class FakePskProvider(params (string Cluster, string Psk)[] keys) : IMeshPskProvider { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/CentralCommandAuthInterceptorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/CentralCommandAuthInterceptorTests.cs index bb619709..510a5dbc 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/CentralCommandAuthInterceptorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/CentralCommandAuthInterceptorTests.cs @@ -248,8 +248,8 @@ public sealed class CentralCommandAuthInterceptorTests // Phase 1A security register item 2. The interceptor must not fetch the raw key and compare // it — that is how a constant-time comparison eventually becomes '=='. Matches owns the // comparison; TryGetPsk survives only for the DIAL side, which has to send the key. - // (The provider's own default Matches calls TryGetPsk internally; this fake overrides it, so - // a zero TryGetPsk count here means the INTERCEPTOR never asked for the key.) + // (The production provider's Matches calls TryGetPsk internally; this fake implements the two + // independently, so a zero TryGetPsk count here means the INTERCEPTOR never asked for the key.) var provider = new FakeMeshPskProvider(("SITE-A", "psk-site-a")); var interceptor = new CentralCommandAuthInterceptor( provider, NullLogger.Instance); @@ -279,8 +279,10 @@ public sealed class CentralCommandAuthInterceptorTests } /// - /// Overrides the interface's default so the two calls are separately countable. The - /// semantics are the default's: no key, or a key that is not exactly the presented token. + /// Implemented independently of so the two calls are separately + /// countable — the interface declares this with no default implementation, deliberately. + /// Semantics match the production provider's: no key, or a key that is not exactly the + /// presented token. /// public bool Matches(string clusterId, string presented) { From 86633f140e7adec6b13ae368b9ae5ebfa168d118 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 6 Aug 2026 06:56:31 -0400 Subject: [PATCH 06/12] test(mesh): pin the Dps no-ClusterClient guard; bound the channel-share test's blackhole worst case --- .../Communication/CentralCommunicationActorTests.cs | 9 ++++++++- .../Communication/MeshGrpcCommandDialerTests.cs | 4 +++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs index a3df8f20..5556295d 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs @@ -735,11 +735,12 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase var factory = NewInMemoryDbFactory(); SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false)); var psk = new RecordingPskProvider(); + var clients = new RecordingClientFactory(() => CreateTestProbe()); Sys.ActorOf(CentralCommunicationActor.Props( factory, new MeshTransportOptions { Mode = mode, ContactRefreshSeconds = 60 }, - new RecordingClientFactory(() => CreateTestProbe()), + clients, () => null, new RecordingDialer(), psk)); @@ -748,6 +749,12 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase () => psk.KnownClusterPublications.Count >= 1, TimeSpan.FromSeconds(5)); psk.KnownClusterPublications[0].ShouldBe(new[] { "C1" }); psk.InvalidateAllCount.ShouldBeGreaterThanOrEqualTo(1); + + // The one load-bearing line the publication itself cannot witness: with the refresh + // timer now running in every mode, only the ClusterClient branch may construct clients. + // With rows seeded, deleting the !_useClusterClient guard turns this red for Dps/Grpc. + if (mode != MeshTransportOptions.ModeClusterClient) + clients.Calls.ShouldBeEmpty(); } [Fact] diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs index be0896aa..f5fd9776 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs @@ -198,7 +198,9 @@ public sealed class MeshGrpcCommandDialerTests // ports on central long before anything else notices. var dialer = new MeshGrpcCommandDialer( new FakePskProvider(("SITE-A", "psk-site-a")), - Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 5 }), + // 1 s, not 5: on a host that blackholes loopback instead of refusing, each dial costs + // the full deadline — keep the worst case bounded so this never reads as a hang. + Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 1 }), NullLogger.Instance); // Nothing is listening, so both dials fail — irrelevant here: the channel is constructed From fbf7b70847c4364a218cacb31825bbe761c1ddc3 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 6 Aug 2026 06:58:34 -0400 Subject: [PATCH 07/12] =?UTF-8?q?docs(plans):=201A.2=20+=201B.2=20close-ou?= =?UTF-8?q?t=20=E2=80=94=20register=20items=201-2=20CLOSED,=20defect=20log?= =?UTF-8?q?=20(3),=20carry-forwards,=20decisions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...26-08-06-clusterclient-to-grpc-tracking.md | 42 ++- ...-08-06-clusterclient-to-grpc.md.tasks.json | 243 ++++++++++++++++-- 2 files changed, 262 insertions(+), 23 deletions(-) diff --git a/docs/plans/2026-08-06-clusterclient-to-grpc-tracking.md b/docs/plans/2026-08-06-clusterclient-to-grpc-tracking.md index 1a7385e5..591b0142 100644 --- a/docs/plans/2026-08-06-clusterclient-to-grpc-tracking.md +++ b/docs/plans/2026-08-06-clusterclient-to-grpc-tracking.md @@ -23,8 +23,8 @@ posture). | Phase | Scope | Branch | Status | PR / merge SHA | Live gate | |---|---|---|---|---|---| | 0 | Proto + mappers + goldens; both auth interceptors; `IMeshPskProvider`; options/validators (+`Grpc` value, SplitTopology relaxation); rig dark config; stale-comment fixes; FleetStatus issue filed | `feat/grpc-phase0-contract-auth` | **MERGED to `master`** @ `10b6d209` (2026-08-06) — all 6 tasks complete @ `23555ed0`; every task spec ✅ + code ✅ (Batch B on Opus with a closed fix loop); issues #542 + #543 filed | `10b6d209` | **PASS 4/4** | -| 1A | Central→node command plane (`NodeCommandService` on the Telemetry listener; `MeshGrpcCommandDialer`; third mode branch in `CentralCommunicationActor`) — **worktree A** | `feat/grpc-node-command` | NOT STARTED | — | PENDING | -| 1B | Node→central ack plane (`CentralCommandService` on the ConfigServe listener; `GrpcApplyAckSender`; third branch in `NodeCommunicationActor`) — **worktree B**, rebases on 1A | `feat/grpc-central-ack` | NOT STARTED | — | PENDING | +| 1A | Central→node command plane (`NodeCommandService` on the Telemetry listener; `MeshGrpcCommandDialer`; third mode branch in `CentralCommunicationActor`) — **worktree A** | `feat/grpc-node-command` | **1A.1 ✅ `64774652` + 1A.2 ✅ `6a862011` (+fixes `48dc2120`/`86633f14`)** — all reviews spec ✅ + code ✅ (1A.2 on Opus, fix loop closed); 1A.3 gate next | — | PENDING | +| 1B | Node→central ack plane (`CentralCommandService` on the ConfigServe listener; `GrpcApplyAckSender`; third branch in `NodeCommunicationActor`) — **worktree B**, rebases on 1A | `feat/grpc-central-ack` | **1B.1 ✅ `14dd07d2` + 1B.2 ✅ `a5a789d4` (+fixes `e54711a2`/`4d607575`)** — all reviews ✅ (1B.2 on Opus: 1 Critical found+fixed, see defect log); awaiting 1A merge → rebase → 1B.3 gate | — | PENDING | | 2 | Baseline reproduction on pre-flip build, then full-rig cutover (`OTOPCUA_MESH_MODE` default → `Grpc`) + soak + kill drills | (rig commit) | NOT STARTED | — | PENDING | | 3 | Delete ClusterClient (code/HOCON/config/tests); SplitTopology requires `Grpc`; grep-gates; docs sweep incl. scadaproj index | `feat/grpc-phase3-deletion` | NOT STARTED | — | — | | 4 | Terminal live gate on the post-deletion image (8 checks, provenance-guarded) | (on `master`) | NOT STARTED | — | PENDING | @@ -46,11 +46,14 @@ phase DoD — PR + merge are user-owned.** | 2026-08-06 | No transport interface — third mode branch inside the two choke-point actors (ScadaBridge §7.1 economy) | design §4.3 | | 2026-08-06 | Coexistence granularity: central's mode governs commands, each node's mode governs its acks (per-pair flips possible — better than ScadaBridge's central-wide flag) | design §3.6 | | 2026-08-06 | `AlarmCommandMessage.operation` is a proto `string`, NOT an enum — the CLR `AlarmCommand.Operation` is an unconstrained string, and an enum would silently decode unknown operations to `UNSPECIFIED` (the drop class this migration removes). Plan T0.1's "AlarmCommand verb" enum wording was a plan defect; only `ApplyOutcome` carries the enum-0 rule. T0.2's goldens must pin pass-through of an unrecognized operation string | proto comment + this log (T0.1 review) | +| 2026-08-06 | **Gate follows the endpoint, not the mode** (1A.2 I-2): known-cluster set published on every contact load in all three modes, from ALL `ClusterNode` rows; fan set unchanged. Survives dark-switch rollback with nothing for 1B to remember | tracking §register refinement + actor docs | +| 2026-08-06 | `CentralCommandService` mapped by PORT, not mode-gated (1B.1): a mode flip on a running fleet must not need a central restart before acks flow; matches every sibling service | `Program.cs` comment + this log | +| 2026-08-06 | Ack deadline is PER ENDPOINT, not per walk; the walk is at-least-once on the deadline path, absorbed idempotently by the coordinator's NodeId-keyed upsert | `MeshTransportOptions` docs + defect log #1 | | 2026-08-06 | `correlation_id` lives on the envelope AND as inner fields on the three commands whose CLR record carries one — each command round-trips independently in the T0.2 mapper; the envelope covers `AlarmCommand` (no CLR correlation member). For the three, the inner field is authoritative on decode | proto comment + this log (T0.1 review) | ## Phase 1A security register (from the Batch B Opus code review — HARD GATE: all four close BEFORE `CentralCommandService` is mapped) -1. **Known-cluster-set membership gate in `TryGetPsk`** — central rebuilds its cluster set from +1. **CLOSED (1A.2 @ `48dc2120`, refined — see below).** Known-cluster-set membership gate in `TryGetPsk` — central rebuilds its cluster set from `ClusterNode` rows every `ContactRefreshSeconds`; refuse a cluster id outside that set in O(1) with no store touch. Closes BOTH open review findings at once: (a) the timing oracle that currently defeats the uniform-denial anti-enumeration text (unknown cluster pays a @@ -61,13 +64,33 @@ phase DoD — PR + merge are user-owned.** audit singleton, attacker-chosen `Target` values, before any credential check). Namespace confinement (`OT-MESH-PSK-` prefix + `SecretName` allow-list) means no disclosure risk — this is flood/oracle only. -2. **`bool Matches(string clusterId, string presented)` on `IMeshPskProvider`** — keep +2. **CLOSED (1A.2 @ `14d21496`).** `bool Matches(string clusterId, string presented)` on `IMeshPskProvider` — keep `FixedTimeEquals` inside the provider so a future verify-side caller cannot write `==` and have it compile. Do before 1B adds the dial-side caller. 3. **DI-shape pin test** — `IMeshPskProvider` is `hasAdmin`-only and its pairing with the central interceptor is by convention; a driver-only node injecting it throws on first call, not at boot. Add a Host.IntegrationTests check (admin host resolves it; driver-only host does NOT) when Phase 1 maps the services. +**Refinement on item 1 (decision, 2026-08-06):** the gate follows the ENDPOINT, not the mode — +published on every contact load in every transport mode (the interceptor and service are +port-mapped, not mode-gated, so a dark-switch rollback must not deactivate the gate), and +built from ALL `ClusterNode` rows (gate answers "does central know this cluster"; the fan set +stays `Enabled && !MaintenanceMode`, so gate ⊇ fan — this also dissolved the +maintenance/ack-auth coupling). Pinned by the every-mode Theory + gate-⊋-fan fact + the Dps +no-ClusterClient guard pin (`86633f14`). Residuals accepted + visible: gate is fail-open until +the first successful contact load (and stays stale if central boots with SQL down — warned +every tick); gate derives from the node table, so a `ServerCluster` with zero rows is outside +it. + +**Follow-ups from the 1A.2/1B.2 Opus reviews (tracked, not blocking):** per-tick +`InvalidateAll` puts a cold-cache 3 s store read on the inbound auth path — MEASURE on the +1B.3 gate (ack burst inside ~3 s after a refresh tick), then consider diff-based +invalidation; the command plane is h2c-only so the per-cluster PSK crosses the network in +cleartext on a write surface — precedent-consistent (all sibling gRPC planes), record as +accepted risk + TLS-option issue (queued below); dialer minors (Lazy channel wrapper, +retired-endpoint eviction, gRPC deadline vs token cancellation, pre-existing +`LoadContactsFromDb` Task.Run(ct) pattern). + 4. **Integration gate over real Kestrel h2c** — both new interceptors have unit coverage only; the one-public-ctor reflection test approximates but cannot prove the interceptor is actually invoked. Wire-probe in the 1A/1B live gates (already in the live-gate tables). @@ -104,7 +127,9 @@ Defects found during execution (live-gate or review), with root cause + pinned t | # | Phase | Defect | Root cause | Fix / pin | |---|---|---|---|---| -| — | — | *(none yet)* | | | +| 1 | 1B.2 | **Failover silently defeated on a hanging central** (CRITICAL, caught by Opus code review): the ack deadline was one absolute `DateTime` shared across the endpoint walk — a hung (not refused) central-1 burned the whole budget and central-2 was dialed with an expired deadline | deadline hoisted outside the `foreach`; a kill-style drill can't see it (refused = fast-fail), only a hang discriminates | `e54711a2`; pinned by `Each_endpoint_gets_a_FRESH_deadline` (mutation-verified); 1B.3 gains a `docker pause` hang drill | +| 2 | 1A.2 | **Dark-switch rollback deactivated the known-cluster gate** (Opus review, I-2): gate published only under `Mode=Grpc` while the interceptor + service are port-mapped in every mode — a rollback to ClusterClient/Dps would silently restore the timing oracle + pre-auth flood | gate activation coupled to transport mode instead of the endpoint it protects | `48dc2120` — gate follows the endpoint; pinned by the every-mode Theory + Dps guard pin `86633f14` | +| 3 | 1A.2 | **Fan parked the actor mailbox** (self-caught by the implementer): dialing from the receive ran `TryGetPsk`'s cold-cache 3 s block pre-await (cold after EVERY refresh due to `InvalidateAll`), stalling deployment acks behind the fan | async methods run synchronously to the first await | `Task.Run` hop (token None — a canceled scheduling would read as a DB failure via PipeTo); pinned red-first by `A_slow_dial_does_not_park_the_actors_mailbox` | ## Carry-forward register @@ -112,13 +137,18 @@ Checks a gate could not drive, named + carried to the next gate (proportionate-g | Item | Deferred from | Proven instead by | Closed in | |---|---|---|---| -| — | — | *(none yet)* | | +| Hang-style central failover drill (`docker pause`, not `docker stop` — only a hang discriminates the per-endpoint-deadline fix) | 1B.2 review | unit fact `Each_endpoint_gets_a_FRESH_deadline` | 1B.3 gate (MUST include) | +| I-4 measurement: ack burst within ~3 s after a contact-refresh tick (cold PSK cache on the inbound auth path) | 1A.2 review | bounded 3 s store read + `Lazy` stampede suppression | 1B.3 gate (observe) | +| `TwoNodeClusterHarness` logs the I-1 null-provider Error at every spawn (harness registers no `IMeshPskProvider`) — EXPECTED, not a rig fault | 1A.2 re-review | production pairing is `hasAdmin`-unconditional in one block | note for 1B.3/Phase 2 readers | +| Register item 3 (DI role-scoping pin: admin resolves provider+dialer, driver-only does NOT) — container-activation test exists, role-scoping pin outstanding | 1A.2 | `It_activates_from_the_container_despite_the_optional_client_factory_parameter` | Phase 2 (or 1B.3) | ## Issues filed by this program | Issue | What | Status | |---|---|---| | [lmxopcua#542](https://gitea.dohertylan.com/dohertj2/lmxopcua/issues/542) | `FleetStatusBroadcaster` derives "fleet" from central's own gossip ring — silently central-pair-blind since mesh Phase 6; pre-existing, transport-independent, out of this program's scope | FILED 2026-08-06 | +| *(to file in 1A.3)* | `ConfigPublishCoordinatorTimeoutTests.DeadlineElapsed_for_current_deployment_marks_TimedOut` is racy on its own terms (150 ms wall-clock deadline; row inserted after spawn) — flaked 2/~11 full runs once 1A.2 grew the suite; green at base. Fix the test (inject clock / raise deadline) | to file | +| *(to file in 1A.3)* | Mesh command/ack planes are h2c-only — the per-cluster PSK crosses the network in cleartext on a write surface (precedent: every sibling gRPC plane; trusted-site-link assumption). Decide: accepted risk, or a `UseTls`/scheme option | to file | | [lmxopcua#543](https://gitea.dohertylan.com/dohertj2/lmxopcua/issues/543) | The three pre-existing auth interceptors (`TelemetryStream`/`ConfigServe`/`LocalDbSync`) treat a whitespace-only key as SET (`IsNullOrEmpty`), so a whitespace `Telemetry:ApiKey` etc. opens the endpoint to a whitespace bearer; the two new mesh interceptors use `IsNullOrWhiteSpace`. Backport is a behaviour change to shipped endpoints — own issue, out of this program's scope | FILED 2026-08-06 | ## Known constraints carried from the research (do not rediscover) diff --git a/docs/plans/2026-08-06-clusterclient-to-grpc.md.tasks.json b/docs/plans/2026-08-06-clusterclient-to-grpc.md.tasks.json index 86202070..aeee0aa2 100644 --- a/docs/plans/2026-08-06-clusterclient-to-grpc.md.tasks.json +++ b/docs/plans/2026-08-06-clusterclient-to-grpc.md.tasks.json @@ -3,22 +3,231 @@ "executionModel": "opus-subagents; sequential within a worktree, 1A and 1B concurrent in separate worktrees (git worktree add ../OtOpcUa-1B feat/grpc-central-ack); 1A merges first, 1B rebases; stop at each phase DoD (PR/merge user-owned)", "worktrees": {}, "tasks": [ - {"id": 0, "subject": "Task 0.1: mesh_transport.proto + contract lock", "phase": "0", "branch": "feat/grpc-phase0-contract-auth", "classification": "standard", "parallelizableWith": ["0.3 (batch, same agent)"], "status": "completed", "commit": "c0ccfa27", "review": "spec ✅ + code ✅"}, - {"id": 1, "subject": "Task 0.2: DTO mapper + round-trip goldens", "phase": "0", "branch": "feat/grpc-phase0-contract-auth", "classification": "high-risk", "parallelizableWith": [], "status": "completed", "blockedBy": [0], "commit": "545413a8", "review": "spec ✅ + code ✅ (serial)"}, - {"id": 2, "subject": "Task 0.3: MeshTransportOptions + validators + SplitTopology relaxation", "phase": "0", "branch": "feat/grpc-phase0-contract-auth", "classification": "standard", "parallelizableWith": ["0.1 (batch, same agent)"], "status": "completed", "commit": "aa21f61f", "review": "spec ✅ + code ✅"}, - {"id": 3, "subject": "Task 0.4: node-side MeshCommandAuthInterceptor", "phase": "0", "branch": "feat/grpc-phase0-contract-auth", "classification": "high-risk", "parallelizableWith": ["0.5 (batch, same agent)"], "status": "completed", "blockedBy": [2], "commit": "fbda015c + fixes 24e55d33/a4631410", "review": "spec ✅ + code ✅ (opus, fix loop closed)"}, - {"id": 4, "subject": "Task 0.5: IMeshPskProvider + CentralCommandAuthInterceptor", "phase": "0", "branch": "feat/grpc-phase0-contract-auth", "classification": "high-risk", "parallelizableWith": ["0.4 (batch, same agent)"], "status": "completed", "blockedBy": [2], "commit": "68a416e4 + fixes 24e55d33/a4631410", "review": "spec ✅ + code ✅ (opus, fix loop closed); 4 items gated on Phase 1A — see tracking §Phase 1A security register"}, - {"id": 5, "subject": "Task 0.6: rig config + FleetStatus issue + docs stub + Phase 0 gate", "phase": "0", "branch": "feat/grpc-phase0-contract-auth", "classification": "small", "parallelizableWith": [], "status": "completed", "blockedBy": [0, 1, 2, 3, 4], "commit": "23555ed0", "review": "code ✅", "gate": "PASS 4/4; issues #542 #543 filed"}, - {"id": 6, "subject": "Task 1A.1: NodeCommandGrpcService on the telemetry listener", "phase": "1A", "branch": "feat/grpc-node-command", "classification": "standard", "parallelizableWith": ["1B.1 (other worktree)"], "status": "completed", "blockedBy": [5], "commit": "64774652", "review": "spec ✅ + code ✅"}, - {"id": 7, "subject": "Task 1A.2: MeshGrpcCommandDialer + third branch in CentralCommunicationActor (+ security register items 1-2)", "phase": "1A", "branch": "feat/grpc-node-command", "classification": "high-risk", "parallelizableWith": ["1B.2 (other worktree)"], "status": "in_progress", "blockedBy": [6]}, - {"id": 8, "subject": "Task 1A.3: 1A live gate (central pair flipped, rig-only, reverted)", "phase": "1A", "branch": "feat/grpc-node-command", "classification": "small", "parallelizableWith": [], "status": "pending", "blockedBy": [7]}, - {"id": 9, "subject": "Task 1B.1: CentralCommandGrpcService + admin validator rule", "phase": "1B", "branch": "feat/grpc-central-ack", "classification": "standard", "parallelizableWith": ["1A.1 (other worktree)"], "status": "pending", "blockedBy": [5]}, - {"id": 10, "subject": "Task 1B.2: GrpcApplyAckSender + node-side wiring", "phase": "1B", "branch": "feat/grpc-central-ack", "classification": "high-risk", "parallelizableWith": ["1A.2 (other worktree)"], "status": "pending", "blockedBy": [9]}, - {"id": 11, "subject": "Task 1B.3: 1B live gate (site-a flipped, rig-only, reverted; rebase on 1A first)", "phase": "1B", "branch": "feat/grpc-central-ack", "classification": "small", "parallelizableWith": [], "status": "pending", "blockedBy": [8, 10]}, - {"id": 12, "subject": "Phase 2: baseline reproduction + full-rig cutover + soak + kill drills", "phase": "2", "branch": "(rig commit)", "classification": "high-risk", "parallelizableWith": [], "status": "pending", "blockedBy": [8, 11]}, - {"id": 13, "subject": "Task 3.1: delete ClusterClient (code + HOCON + config + tests, ordered)", "phase": "3", "branch": "feat/grpc-phase3-deletion", "classification": "high-risk", "parallelizableWith": [], "status": "pending", "blockedBy": [12]}, - {"id": 14, "subject": "Task 3.2: docs sweep (Configuration/Redundancy/Telemetry/CLAUDE.md + scadaproj index)", "phase": "3", "branch": "feat/grpc-phase3-deletion", "classification": "small", "parallelizableWith": [], "status": "pending", "blockedBy": [13]}, - {"id": 15, "subject": "Phase 4: terminal live gate on post-deletion image (8 checks, provenance-guarded)", "phase": "4", "branch": "(master)", "classification": "high-risk", "parallelizableWith": [], "status": "pending", "blockedBy": [14]} + { + "id": 0, + "subject": "Task 0.1: mesh_transport.proto + contract lock", + "phase": "0", + "branch": "feat/grpc-phase0-contract-auth", + "classification": "standard", + "parallelizableWith": [ + "0.3 (batch, same agent)" + ], + "status": "completed", + "commit": "c0ccfa27", + "review": "spec ✅ + code ✅" + }, + { + "id": 1, + "subject": "Task 0.2: DTO mapper + round-trip goldens", + "phase": "0", + "branch": "feat/grpc-phase0-contract-auth", + "classification": "high-risk", + "parallelizableWith": [], + "status": "completed", + "blockedBy": [ + 0 + ], + "commit": "545413a8", + "review": "spec ✅ + code ✅ (serial)" + }, + { + "id": 2, + "subject": "Task 0.3: MeshTransportOptions + validators + SplitTopology relaxation", + "phase": "0", + "branch": "feat/grpc-phase0-contract-auth", + "classification": "standard", + "parallelizableWith": [ + "0.1 (batch, same agent)" + ], + "status": "completed", + "commit": "aa21f61f", + "review": "spec ✅ + code ✅" + }, + { + "id": 3, + "subject": "Task 0.4: node-side MeshCommandAuthInterceptor", + "phase": "0", + "branch": "feat/grpc-phase0-contract-auth", + "classification": "high-risk", + "parallelizableWith": [ + "0.5 (batch, same agent)" + ], + "status": "completed", + "blockedBy": [ + 2 + ], + "commit": "fbda015c + fixes 24e55d33/a4631410", + "review": "spec ✅ + code ✅ (opus, fix loop closed)" + }, + { + "id": 4, + "subject": "Task 0.5: IMeshPskProvider + CentralCommandAuthInterceptor", + "phase": "0", + "branch": "feat/grpc-phase0-contract-auth", + "classification": "high-risk", + "parallelizableWith": [ + "0.4 (batch, same agent)" + ], + "status": "completed", + "blockedBy": [ + 2 + ], + "commit": "68a416e4 + fixes 24e55d33/a4631410", + "review": "spec ✅ + code ✅ (opus, fix loop closed); 4 items gated on Phase 1A — see tracking §Phase 1A security register" + }, + { + "id": 5, + "subject": "Task 0.6: rig config + FleetStatus issue + docs stub + Phase 0 gate", + "phase": "0", + "branch": "feat/grpc-phase0-contract-auth", + "classification": "small", + "parallelizableWith": [], + "status": "completed", + "blockedBy": [ + 0, + 1, + 2, + 3, + 4 + ], + "commit": "23555ed0", + "review": "code ✅", + "gate": "PASS 4/4; issues #542 #543 filed" + }, + { + "id": 6, + "subject": "Task 1A.1: NodeCommandGrpcService on the telemetry listener", + "phase": "1A", + "branch": "feat/grpc-node-command", + "classification": "standard", + "parallelizableWith": [ + "1B.1 (other worktree)" + ], + "status": "completed", + "blockedBy": [ + 5 + ], + "commit": "64774652", + "review": "spec ✅ + code ✅" + }, + { + "id": 7, + "subject": "Task 1A.2: MeshGrpcCommandDialer + third branch in CentralCommunicationActor (+ security register items 1-2)", + "phase": "1A", + "branch": "feat/grpc-node-command", + "classification": "high-risk", + "parallelizableWith": [ + "1B.2 (other worktree)" + ], + "status": "completed", + "blockedBy": [ + 6 + ], + "commit": "14d21496+6a862011 + fixes 48dc2120/86633f14", + "review": "spec ✅ + code ✅ (opus; I-2 fixed via gate-follows-endpoint decision)" + }, + { + "id": 8, + "subject": "Task 1A.3: 1A live gate (central pair flipped, rig-only, reverted)", + "phase": "1A", + "branch": "feat/grpc-node-command", + "classification": "small", + "parallelizableWith": [], + "status": "in_progress", + "blockedBy": [ + 7 + ] + }, + { + "id": 9, + "subject": "Task 1B.1: CentralCommandGrpcService + admin validator rule", + "phase": "1B", + "branch": "feat/grpc-central-ack", + "classification": "standard", + "parallelizableWith": [ + "1A.1 (other worktree)" + ], + "status": "pending", + "blockedBy": [ + 5 + ] + }, + { + "id": 10, + "subject": "Task 1B.2: GrpcApplyAckSender + node-side wiring", + "phase": "1B", + "branch": "feat/grpc-central-ack", + "classification": "high-risk", + "parallelizableWith": [ + "1A.2 (other worktree)" + ], + "status": "pending", + "blockedBy": [ + 9 + ] + }, + { + "id": 11, + "subject": "Task 1B.3: 1B live gate (site-a flipped, rig-only, reverted; rebase on 1A first)", + "phase": "1B", + "branch": "feat/grpc-central-ack", + "classification": "small", + "parallelizableWith": [], + "status": "pending", + "blockedBy": [ + 8, + 10 + ] + }, + { + "id": 12, + "subject": "Phase 2: baseline reproduction + full-rig cutover + soak + kill drills", + "phase": "2", + "branch": "(rig commit)", + "classification": "high-risk", + "parallelizableWith": [], + "status": "pending", + "blockedBy": [ + 8, + 11 + ] + }, + { + "id": 13, + "subject": "Task 3.1: delete ClusterClient (code + HOCON + config + tests, ordered)", + "phase": "3", + "branch": "feat/grpc-phase3-deletion", + "classification": "high-risk", + "parallelizableWith": [], + "status": "pending", + "blockedBy": [ + 12 + ] + }, + { + "id": 14, + "subject": "Task 3.2: docs sweep (Configuration/Redundancy/Telemetry/CLAUDE.md + scadaproj index)", + "phase": "3", + "branch": "feat/grpc-phase3-deletion", + "classification": "small", + "parallelizableWith": [], + "status": "pending", + "blockedBy": [ + 13 + ] + }, + { + "id": 15, + "subject": "Phase 4: terminal live gate on post-deletion image (8 checks, provenance-guarded)", + "phase": "4", + "branch": "(master)", + "classification": "high-risk", + "parallelizableWith": [], + "status": "pending", + "blockedBy": [ + 14 + ] + } ], "lastUpdated": "2026-08-06" -} +} \ No newline at end of file From 410c2e2925bfa6570070230c394a8ae34e736a97 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 6 Aug 2026 07:38:18 -0400 Subject: [PATCH 08/12] docs(mesh): Phase 1A live gate record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ran the 1A live gate on the docker-dev rig at fbf7b708 (image rebuilt from this worktree, all eight containers recreated). VERDICT: FAIL. Both central nodes fail to boot on this branch with MissingActorRegistryEntryException: AuditWriterActorKey — the first WithActors starter eagerly resolves IMeshCommandDialer/IMeshPskProvider, whose graph reaches ActorAuditWriter, whose ctor reads an ActorRegistry key registered ~60 lines later in the same builder. Reproduced on the same image with MeshTransport__Mode=ClusterClient, so it is mode-independent and not caused by the gate's rig-only flip. Recorded as defect 4; no product change made. Checks 5 (auth negative/positive on real Kestrel h2c) and 6 (wrong-key isolation, both directions, with a positive control) PASS. Checks 3 and 4 pass their node-side leg only, driven with real RestartDriver/ReconnectDriver/ AlarmCommand envelopes that restarted and reconnected SITEA-modbus. Checks 1, 2 and 7 are blocked outright — no central, no dispatch, no contact refresh — and check 8 is vacuous for telemetry + config-serve because central dials both and never booted. All five gaps are named in the carry-forward register and must be closed by a 1A gate re-run, not carried past the phase. Issues filed by this task: lmxopcua#544 (ConfigPublishCoordinatorTimeoutTests deflake) and lmxopcua#545 (mesh planes are h2c-only — accept the risk or add a UseTls option); the tracking table's two placeholder rows now carry them. Also recorded: DriverStatusPanel.razor is referenced by no page or component, so the AdminUI Reconnect/Restart controls are unreachable on this branch and on master — pre-existing, out of program scope, carried forward. Rig restored: compose flip reverted, six nodes healthy on all-ClusterClient. The rig runs the PRE-gate image (the 1A build is kept as otopcua-host:grpc-1a); a re-run must rebuild. --- ...6-08-06-clusterclient-to-grpc-live-gate.md | 232 +++++++++++++++++- ...26-08-06-clusterclient-to-grpc-tracking.md | 12 +- 2 files changed, 227 insertions(+), 17 deletions(-) diff --git a/docs/plans/2026-08-06-clusterclient-to-grpc-live-gate.md b/docs/plans/2026-08-06-clusterclient-to-grpc-live-gate.md index 9dc56776..d1811c24 100644 --- a/docs/plans/2026-08-06-clusterclient-to-grpc-live-gate.md +++ b/docs/plans/2026-08-06-clusterclient-to-grpc-live-gate.md @@ -154,22 +154,217 @@ Rig left running on `ClusterClient` (its normal state) after the gate. --- -## Phase 1A — central→node command plane — **PENDING** +## Phase 1A — central→node command plane — **FAIL (blocked at the central boot boundary)** -Branch `feat/grpc-node-command` @ `________`. Rig-only flip: central-1/-2 → -`MeshTransport__Mode=Grpc`; **site nodes remain `ClusterClient`** (coexistence is the point). -Flip reverted after the gate, never committed. +Branch `feat/grpc-node-command` @ `fbf7b708`. Image rebuilt from this worktree +(`/Users/dohertj2/Desktop/OtOpcUa/.worktrees/grpc-1a`) so it carries this branch; +`docker compose -f docker-dev/docker-compose.yml up -d --build` exited 0 and **all eight +containers were recreated** — `migrator` and `cluster-seed` re-ran to completion (exit 0), the +six host nodes and traefik were recreated, the `sql` container and its named volume were kept. +Nothing else touched the stack afterwards, so this run is not a concurrent-rebuild-invalidated +one. Rig-only flip: central-1/-2 `MeshTransport__Mode: "Grpc"` (hard value); **site nodes left +on the compose default `ClusterClient`** (coexistence is the point). Three Serilog +`MinimumLevel:Override` Debug scopes were added on all six nodes for this run only +(`ZB.MOM.WW.OtOpcUa.Host.Grpc`, `ZB.MOM.WW.OtOpcUa.ControlPlane.Communication`, +`Akka.Cluster.Tools.Client`) — the accept/dispatch lines the checks read are `Debug` and the +rig's default Serilog level is `Information`, so without them checks 1/3 would have had no +observable signal. Flip and overrides reverted after the gate, never committed. + +**Verdict: FAIL.** Neither central node boots on this branch, so nothing that originates at +central could be driven. Root cause is defect 4 below — resolving `IMeshCommandDialer` / +`IMeshPskProvider` inside the first `WithActors` starter drags in `ISecretResolver` → +`IAuditWriter` → `ActorAuditWriter`, whose constructor reads `AuditWriterActorKey` out of the +`ActorRegistry` before the singleton that registers it has run. It is **mode-independent** +(reproduced on the same image with `MeshTransport__Mode=ClusterClient`), so it is a defect of +the branch, not of the gate's flip, and it would have failed a `ClusterClient` rig equally. | # | Check | Result | |---|---|---| -| 1 | Deploy dispatch from AdminUI reaches **all six** driver nodes over gRPC (per-node `Execute` log lines on every node; zero `SendToAll` lines on central) | PENDING | -| 2 | Acks still return over ClusterClient (site mode unchanged) and the deployment **seals** — mixed-transport deploy proven end to end | PENDING | -| 3 | `RestartDriver` + `ReconnectDriver` from AdminUI land on the target site node (EventStream delivery observed via driver restart in logs) | PENDING | -| 4 | `AlarmCommand` (ack a scripted alarm from AdminUI) lands and the condition acknowledges | PENDING | -| 5 | Unauthenticated `grpcurl` to a site node `:4056` `mesh_transport.v1.NodeCommandService/Execute` ⇒ `PermissionDenied`; correct dev PSK ⇒ accepted | PENDING | -| 6 | **Wrong-key isolation:** site-b's PSK presented to site-a ⇒ `PermissionDenied` (per-cluster scoping is real, not decorative) | PENDING | -| 7 | A node with NULL `GrpcPort` is skipped with the warn-once, fan continues to the others | PENDING | -| 8 | Telemetry + config-serve + LocalDb sync unaffected: 0 auth failures on their interceptors across the run | PENDING | +| 1 | Deploy dispatch from AdminUI reaches **all six** driver nodes over gRPC (per-node `Execute` log lines on every node; zero `SendToAll` lines on central) | **BLOCKED** — `POST /api/deployments` is unreachable: traefik answers `503` because neither central node has a live HTTP surface (`curl http://central-1:9000/health/ready` and `central-2` both return `000`, connection refused). Nothing dispatched. | +| 2 | Acks still return over ClusterClient (site mode unchanged) and the deployment **seals** — mixed-transport deploy proven end to end | **BLOCKED** — no deployment could be dispatched (check 1). The most recent `dbo.Deployment` row is still the pre-gate `03540DA5-…` (`Status=2`, Sealed, `2026-08-06 06:11:22`), i.e. nothing was attempted during this run. | +| 3 | `RestartDriver` + `ReconnectDriver` from AdminUI land on the target site node (EventStream delivery observed via driver restart in logs) | **PARTIAL PASS (node-side leg only)** — both commands were delivered to site-a-1 over real Kestrel h2c with the correct PSK and both acted: `DriverHost site-a-1:4053: restarting driver SITEA-modbus by request of 1a-gate` followed by a fresh `spawned Modbus driver SITEA-modbus`, and `reconnecting driver … ForceReconnect requested by admin; re-entering Reconnecting`. The **AdminUI half was not driven** — see the two carry-forwards below (central down; and the UI control is unreachable on this branch anyway). | +| 4 | `AlarmCommand` (ack a scripted alarm from AdminUI) lands and the condition acknowledges | **PARTIAL PASS (envelope leg only)** — the alarm arm decoded and was delivered node-side (`Mesh: accepted AlarmCommand (correlationId=39ba4511…) over gRPC`), which is the arm with no CLR correlation member and therefore the one worth wiring live. **The acknowledge itself was NOT observed**: the rig's single `dbo.ScriptedAlarm` row (`gate481-alarm`, equipment `EQ-6e09bcc2d907`) is not owned by SITE-A, and its owning node is a central one, which is down. | +| 5 | Unauthenticated `grpcurl` to a site node `:4056` `mesh_transport.v1.NodeCommandService/Execute` ⇒ `PermissionDenied`; correct dev PSK ⇒ accepted | **PASS** — 3/3 probes graded as designed: no bearer ⇒ `PermissionDenied`, correct `dev-mesh-psk-SITE-A` + empty envelope ⇒ `InvalidArgument` (**auth passed**, the decode is what failed), and a full `RestartDriver` envelope with the same key ⇒ `{}` + observed driver restart. | +| 6 | **Wrong-key isolation:** site-b's PSK presented to site-a ⇒ `PermissionDenied` (per-cluster scoping is real, not decorative) | **PASS** — both directions rejected: `dev-mesh-psk-SITE-B` → site-a-1 ⇒ `PermissionDenied`, `dev-mesh-psk-SITE-A` → site-b-1 ⇒ `PermissionDenied`, while `dev-mesh-psk-SITE-B` → site-b-1 reaches the decoder (`InvalidArgument`) — so the rejection is the key, not the reachability. | +| 7 | A node with NULL `GrpcPort` is skipped with the warn-once, fan continues to the others | **BLOCKED** — the skip is computed by `CentralCommunicationActor.AdoptGrpcTargets` on a contact refresh; no central node is running to refresh. The rig's `ClusterNode` rows were left untouched (all six still `GrpcPort=4056`) rather than mutated for a check that could not be observed. | +| 8 | Telemetry + config-serve + LocalDb sync unaffected: 0 auth failures on their interceptors across the run | **PARTIAL PASS** — 0 `Rejected a telemetry stream call` / `Rejected a config-serve call` / `Rejected a LocalDb sync call` lines across all six nodes, and the only `Rejected a mesh command call` lines are the 3 deliberate negative probes (2 on site-a-1, 1 on site-b-1). **But telemetry and config-serve carried no traffic at all** in this run — central dials both and never booted (site-a-1 logs 0 telemetry lines) — so "unaffected" is established for LocalDb sync only. | + +### Phase 1A evidence + +**Provenance** — flip in effect on central only, sites untouched: + +``` +$ for c in central-1 central-2 site-a-1 site-b-1; do docker inspect otopcua-dev-$c-1 \ + --format '{{range .Config.Env}}{{println .}}{{end}}' | grep '^MeshTransport__Mode='; done +central-1 MeshTransport__Mode=Grpc +central-2 MeshTransport__Mode=Grpc +site-a-1 MeshTransport__Mode=ClusterClient +site-b-1 MeshTransport__Mode=ClusterClient + +$ docker images --format '{{.Repository}}:{{.Tag}}|{{.ID}}|{{.CreatedAt}}' | grep otopcua-host +otopcua-host:dev|69360a173f1f|2026-08-06 07:20:14 -0400 EDT # rebuilt for this run +``` + +**Defect 4 — the boot failure (checks 1/2/7 blocked by it).** Both central nodes, identical: + +``` +$ docker logs otopcua-dev-central-1-1 2>&1 | sed -n '25,45p' +[11:21:27 INF] OPC UA server started on opc.tcp://0.0.0.0:4840 +[11:21:27 FTL] Unable to start AkkaHostedService - shutting down application +Akka.Hosting.MissingActorRegistryEntryException: No actor registered for key + ZB.MOM.WW.OtOpcUa.ControlPlane.AuditWriterActorKey + at Akka.Hosting.ActorRegistry.Get[TKey]() + at ZB.MOM.WW.OtOpcUa.AdminUI.Audit.ActorAuditWriter..ctor(ActorRegistry registry) + in /src/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Audit/ActorAuditWriter.cs:line 24 + ... + at ZB.MOM.WW.Secrets.DependencyInjection.SecretsServiceCollectionExtensions.<>c.b__0_5(IServiceProvider sp) + ... + at ZB.MOM.WW.Secrets.DependencyInjection.SecretsServiceCollectionExtensions.<>c.b__0_6(IServiceProvider sp) + ... + at ZB.MOM.WW.OtOpcUa.ControlPlane.ServiceCollectionExtensions.<>c.b__7_0( + ActorSystem system, IActorRegistry registry, IDependencyResolver resolver) + in /src/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs:line 74 + at Akka.Hosting.AkkaConfigurationBuilder.StartAsync(ActorSystem sys) + at Akka.Hosting.AkkaHostedService.StartAsync(CancellationToken cancellationToken) +``` + +**Mode-independence, proven rather than argued** — same image, central-1's exact environment +with `MeshTransport__Mode=ClusterClient`, own hostname + self-seed so it cannot disturb the rig: + +``` +$ docker run -d --name scratch-cc-mode --network otopcua-dev_default --env-file c1-env.txt \ + -e Cluster__PublicHostname=scratch-cc-mode \ + -e Cluster__SeedNodes__0=akka.tcp://otopcua@scratch-cc-mode:4053 otopcua-host:dev +$ docker logs scratch-cc-mode 2>&1 | grep -m4 -E "MissingActorRegistry|Hosting failed|Host starting" +[11:28:45 INF] OtOpcUa.Host starting with roles=[admin,driver,cluster-main] (admin=True, driver=True) +Akka.Hosting.MissingActorRegistryEntryException: No actor registered for key + ZB.MOM.WW.OtOpcUa.ControlPlane.AuditWriterActorKey +[11:28:47 ERR] Hosting failed to start +``` + +Scratch container removed after capture (`docker rm -f scratch-cc-mode`); the six real nodes +were untouched by it. + +**It presents as a healthy container.** Worth carrying into Phase 2/4 readings — the failed +process does not exit, so `docker ps` shows `Up`, `RestartCount=0`, and only an HTTP probe or +the log tells the truth: + +``` +$ docker inspect otopcua-dev-central-1-1 -f '{{.State.Status}} restarts={{.RestartCount}}' +running restarts=0 +$ docker run --rm --network otopcua-dev_default curlimages/curl -s -o /dev/null \ + -w '%{http_code}\n' --max-time 5 http://central-1:9000/health/ready +000 +$ curl -s -o /dev/null -w '%{http_code}\n' http://localhost:9200/health/ready +503 +``` + +**Checks 5 + 6 — the wire-probe auth matrix** (`-import-path` onto the in-repo proto; the node +has no reflection). An empty `CommandEnvelope` has no `oneof` arm set, so `FromEnvelope` throws +and the service answers `InvalidArgument` — **after** the interceptor ran, which is exactly why +it is the positive signal here: `PermissionDenied` means the key was refused, `InvalidArgument` +means the key was accepted and only the payload was rejected: + +``` +$ P=src/Core/ZB.MOM.WW.OtOpcUa.Commons/Protos +$ G="docker run --rm --network otopcua-dev_default -v $PWD/$P:/protos:ro fullstorydev/grpcurl \ + -plaintext -import-path /protos -proto mesh_transport.proto" + +# 5a no bearer -> site-a-1 +Code: PermissionDenied Message: Node command authentication failed. +# 5b -H 'authorization: Bearer dev-mesh-psk-SITE-A' -> site-a-1 +Code: InvalidArgument Message: Command envelope could not be decoded. +# 6a -H 'authorization: Bearer dev-mesh-psk-SITE-B' -> site-a-1 +Code: PermissionDenied Message: Node command authentication failed. +# 6b -H 'authorization: Bearer dev-mesh-psk-SITE-A' -> site-b-1 +Code: PermissionDenied Message: Node command authentication failed. +# 6c -H 'authorization: Bearer dev-mesh-psk-SITE-B' -> site-b-1 (positive control) +Code: InvalidArgument Message: Command envelope could not be decoded. +``` + +**Checks 3 + 4 — the node-side leg, driven with real envelopes** over the same authenticated +h2c path central would use. This is the first live proof that `MeshCommandAuthInterceptor` is +actually invoked on a real Kestrel pipeline (register item 4) and that +`NodeCommandGrpcService` → `NodeCommunicationActor` → EventStream → `DriverHostActor` works +end to end: + +``` +$ $G -H 'authorization: Bearer dev-mesh-psk-SITE-A' -H 'x-otopcua-cluster: SITE-A' \ + -d '{"correlationId":"8461008e...","restartDriver":{"clusterId":"SITE-A", + "driverInstanceId":"SITEA-modbus","actorByUserName":"1a-gate", + "correlationId":"8461008e..."}}' \ + site-a-1:4056 mesh_transport.v1.NodeCommandService/Execute +{} + +$ docker logs otopcua-dev-site-a-1-1 2>&1 | tail +[11:31:01 DBG] Mesh: accepted RestartDriver (correlationId=8461008e37654724a292219202d78dfa) over gRPC; handed to the node communication actor. +[11:31:01 INF] DriverHost site-a-1:4053: restarting driver SITEA-modbus by request of 1a-gate +[11:31:01 INF] DriverHost site-a-1:4053: spawned Modbus driver SITEA-modbus (stub=False) +[11:31:55 DBG] Mesh: accepted ReconnectDriver (correlationId=29c5dc77283f4eb7869d0c7d6686cadd) over gRPC; handed to the node communication actor. +[11:31:55 INF] DriverHost site-a-1:4053: reconnecting driver SITEA-modbus by request of 1a-gate +[11:31:55 INF] DriverInstance SITEA-modbus: ForceReconnect requested by admin; re-entering Reconnecting +[11:31:56 DBG] Mesh: accepted AlarmCommand (correlationId=39ba451173954229a71b9dd1e00ac1b9) over gRPC; handed to the node communication actor. +``` + +**Check 8 — interceptor denial counts across every node, whole run:** + +``` +$ for c in central-1 central-2 site-a-1 site-a-2 site-b-1 site-b-2; do … done +central-1 telemetry=0 configserve=0 localdbsync=0 mesh=0 +central-2 telemetry=0 configserve=0 localdbsync=0 mesh=0 +site-a-1 telemetry=0 configserve=0 localdbsync=0 mesh=2 # the two deliberate negatives +site-a-2 telemetry=0 configserve=0 localdbsync=0 mesh=0 +site-b-1 telemetry=0 configserve=0 localdbsync=0 mesh=1 # the one deliberate negative +site-b-2 telemetry=0 configserve=0 localdbsync=0 mesh=0 +``` + +The `TwoNodeClusterHarness` I-1 null-provider Error (a known test-harness artifact) did **not** +appear in any rig log — `grep -c "IMeshPskProvider"` = 0 on all six nodes, as expected for +production wiring where every admin node registers the provider. + +### Phase 1A defect + +**Defect 4 — an admin node cannot boot: eager DI resolution inside the first Akka starter.** +`WithOtOpcUaControlPlaneSingletons`'s `WithActors` block resolves `IMeshCommandDialer` and +`IMeshPskProvider` from `IServiceProvider` to hand to `CentralCommunicationActor.Props`. +`MeshGrpcCommandDialer` takes `IMeshPskProvider`, which takes `ISecretResolver`, which +`AddZbSecrets` builds by resolving `IAuditWriter`, which on an admin node is +`ActorAuditWriter` — and its constructor does `registry.Get()`. That +singleton is registered ~60 lines later in the same builder, so at this point the key is +absent and `ActorRegistry.Get` throws `MissingActorRegistryEntryException`, failing +`AkkaHostedService.StartAsync` and the whole host. + +Three things make this worth a pinned regression rather than a one-line fix note: + +- **It is not mode-gated.** The two `GetService` calls run in every transport mode by design + (so a mode flip needs no restart), so the eager graph is walked on every admin boot. Proven + above with a `ClusterClient` scratch container on the same image. +- **It is invisible to the suite.** The 1A.2 container-activation test + (`It_activates_from_the_container_despite_the_optional_client_factory_parameter`) resolves + the dialer from a service collection that does not include `AddZbSecrets`, and + `TwoNodeClusterHarness` registers no `IMeshPskProvider` at all — so no existing test walks + the real graph in starter order. This is the same class as scadaproj#1 ("only visible with + the DI graph built in a real host") and as mesh Phase 6's `IClusterRoleInfo` boot crash. +- **The container lies about it.** The process does not exit, so `docker ps` reports `Up` with + `restarts=0` while the node serves no HTTP at all. Any Phase 2/4 readiness check that reads + container status rather than `/health/ready` would score this as healthy. + +The shape of the fix is the one this file already uses two lines above for the coordinator +proxy — hand the actor a lazy accessor (`Func` / `Func`, +or resolve on first use) so nothing is constructed while the `ActorSystem` is still building — +but the fix and its red-first pin belong to 1A, not to this record. + +### Rig state after the gate + +The compose flip was reverted (`git checkout docker-dev/docker-compose.yml`) and the six nodes +recreated on the committed all-`ClusterClient` compose. Because defect 4 is mode-independent, +the branch image cannot produce a healthy central pair on *any* mode, so the rig was returned +to its pre-gate image rather than left with a dead central pair: the 1A build was preserved as +`otopcua-host:grpc-1a` (`69360a173f1f`) and the pre-gate build (`8bd81d8aea9b`, 04:52) re-tagged +back to `otopcua-host:dev`. **Anyone re-running this gate must rebuild — the running rig is +NOT on the branch under test.** Verified after restore: six nodes `running restarts=0`, zero +`Hosting failed to start` on all six, `/health/ready` 200 on every node, MAIN mesh +`memberCount=2 unreachableCount=0`, AdminUI 200 on `http://localhost:9200`. --- @@ -228,8 +423,19 @@ checks 6–7. ### Carry-forwards Anything a phase gate could not drive gets named here with why + what proves it instead, and -is re-checked in the next gate (the ScadaBridge proportionate-gate rule). Currently: none. +is re-checked in the next gate (the ScadaBridge proportionate-gate rule). + +| Item | Deferred from | Why it could not be driven | Proven instead by | Re-check in | +|---|---|---|---|---| +| Checks 1, 2, 7 in full — deploy dispatch over gRPC, mixed-transport seal, NULL-`GrpcPort` skip | 1A | Both central nodes fail to boot (defect 4); no AdminUI, no deploy API, no contact refresh | Nothing live. Unit coverage only: the fan Theory, the gate-⊋-fan fact, `A_slow_dial_does_not_park_the_actors_mailbox` | **1A gate re-run after defect 4 is fixed** — these are the checks 1A exists to prove and must not be carried past it | +| Check 3 central half — `RestartDriver`/`ReconnectDriver` **originating at the AdminUI** | 1A | (a) central down; (b) independently, `DriverStatusPanel.razor` — the only component holding the Reconnect/Restart buttons — is referenced by **no page or component in the AdminUI**, so the control is unreachable in the running UI on this branch and on `master` | The node-side leg driven live with real envelopes (check 3 evidence); central's encode covered by the T0.2 mapper goldens | 1A re-run; the orphaned panel is pre-existing and out of this program's scope — file separately if it is to be fixed | +| Check 4 acknowledge half — the condition actually acknowledging | 1A | The rig's one `ScriptedAlarm` (`gate481-alarm`) is not owned by a SITE cluster, and its owning central node is down | The alarm arm decoding + delivering node-side (check 4 evidence) | 1A re-run, seeding a SITE-A-owned scripted alarm first | +| Check 8 for telemetry + config-serve | 1A | Central dials both planes and never booted, so neither carried a single call — 0 denials is vacuous for them | LocalDb sync (pair-local) did carry traffic with 0 denials | 1A re-run | ### Defects the gates caught that the suite did not Recorded per phase as they occur, with root cause + the pinned regression test. + +| # | Phase | Defect | Root cause | Status | +|---|---|---|---|---| +| 4 | 1A gate | **An admin node cannot boot at all** — `MissingActorRegistryEntryException: AuditWriterActorKey`, both central nodes, every transport mode | The first `WithActors` starter eagerly resolves `IMeshCommandDialer`/`IMeshPskProvider`, whose graph reaches `ActorAuditWriter`, whose ctor reads an `ActorRegistry` key registered ~60 lines later in the same builder | **OPEN — blocks 1A.** Full analysis + evidence in the Phase 1A section above. No product change was made by this gate | diff --git a/docs/plans/2026-08-06-clusterclient-to-grpc-tracking.md b/docs/plans/2026-08-06-clusterclient-to-grpc-tracking.md index 591b0142..89709602 100644 --- a/docs/plans/2026-08-06-clusterclient-to-grpc-tracking.md +++ b/docs/plans/2026-08-06-clusterclient-to-grpc-tracking.md @@ -23,7 +23,7 @@ posture). | Phase | Scope | Branch | Status | PR / merge SHA | Live gate | |---|---|---|---|---|---| | 0 | Proto + mappers + goldens; both auth interceptors; `IMeshPskProvider`; options/validators (+`Grpc` value, SplitTopology relaxation); rig dark config; stale-comment fixes; FleetStatus issue filed | `feat/grpc-phase0-contract-auth` | **MERGED to `master`** @ `10b6d209` (2026-08-06) — all 6 tasks complete @ `23555ed0`; every task spec ✅ + code ✅ (Batch B on Opus with a closed fix loop); issues #542 + #543 filed | `10b6d209` | **PASS 4/4** | -| 1A | Central→node command plane (`NodeCommandService` on the Telemetry listener; `MeshGrpcCommandDialer`; third mode branch in `CentralCommunicationActor`) — **worktree A** | `feat/grpc-node-command` | **1A.1 ✅ `64774652` + 1A.2 ✅ `6a862011` (+fixes `48dc2120`/`86633f14`)** — all reviews spec ✅ + code ✅ (1A.2 on Opus, fix loop closed); 1A.3 gate next | — | PENDING | +| 1A | Central→node command plane (`NodeCommandService` on the Telemetry listener; `MeshGrpcCommandDialer`; third mode branch in `CentralCommunicationActor`) — **worktree A** | `feat/grpc-node-command` | **1A.3 gate RAN 2026-08-06 @ `fbf7b708` and FAILED — defect 4 blocks the phase.** Both central nodes fail to boot (mode-independent), so checks 1/2/7 could not be driven at all and 3/4 only node-side; 5/6 PASS. 1A.1 ✅ `64774652` + 1A.2 ✅ `6a862011` (+fixes `48dc2120`/`86633f14`) all reviewed spec ✅ + code ✅. **Next: fix defect 4 with a red-first pin, then RE-RUN the 1A gate** | — | **FAIL** (5 of 8 not provable; 2 PASS, 2 partial, 3 blocked) | | 1B | Node→central ack plane (`CentralCommandService` on the ConfigServe listener; `GrpcApplyAckSender`; third branch in `NodeCommunicationActor`) — **worktree B**, rebases on 1A | `feat/grpc-central-ack` | **1B.1 ✅ `14dd07d2` + 1B.2 ✅ `a5a789d4` (+fixes `e54711a2`/`4d607575`)** — all reviews ✅ (1B.2 on Opus: 1 Critical found+fixed, see defect log); awaiting 1A merge → rebase → 1B.3 gate | — | PENDING | | 2 | Baseline reproduction on pre-flip build, then full-rig cutover (`OTOPCUA_MESH_MODE` default → `Grpc`) + soak + kill drills | (rig commit) | NOT STARTED | — | PENDING | | 3 | Delete ClusterClient (code/HOCON/config/tests); SplitTopology requires `Grpc`; grep-gates; docs sweep incl. scadaproj index | `feat/grpc-phase3-deletion` | NOT STARTED | — | — | @@ -129,6 +129,7 @@ Defects found during execution (live-gate or review), with root cause + pinned t |---|---|---|---|---| | 1 | 1B.2 | **Failover silently defeated on a hanging central** (CRITICAL, caught by Opus code review): the ack deadline was one absolute `DateTime` shared across the endpoint walk — a hung (not refused) central-1 burned the whole budget and central-2 was dialed with an expired deadline | deadline hoisted outside the `foreach`; a kill-style drill can't see it (refused = fast-fail), only a hang discriminates | `e54711a2`; pinned by `Each_endpoint_gets_a_FRESH_deadline` (mutation-verified); 1B.3 gains a `docker pause` hang drill | | 2 | 1A.2 | **Dark-switch rollback deactivated the known-cluster gate** (Opus review, I-2): gate published only under `Mode=Grpc` while the interceptor + service are port-mapped in every mode — a rollback to ClusterClient/Dps would silently restore the timing oracle + pre-auth flood | gate activation coupled to transport mode instead of the endpoint it protects | `48dc2120` — gate follows the endpoint; pinned by the every-mode Theory + Dps guard pin `86633f14` | +| 4 | 1A.3 (live gate) | **An admin node cannot boot** (BLOCKS 1A): both central nodes die at startup with `MissingActorRegistryEntryException: ZB.MOM.WW.OtOpcUa.ControlPlane.AuditWriterActorKey`, taking `AkkaHostedService` and the whole host with them. Mode-INDEPENDENT — reproduced on the same image with `MeshTransport__Mode=ClusterClient`, so it is not the gate's flip. Driver-only nodes are unaffected (they register neither `IMeshPskProvider` nor `ActorAuditWriter`) | `WithOtOpcUaControlPlaneSingletons`'s first `WithActors` starter eagerly resolves `IMeshCommandDialer` + `IMeshPskProvider` to pass into `CentralCommunicationActor.Props`. That graph is dialer → `IMeshPskProvider` → `ISecretResolver` → (`AddZbSecrets`) → `IAuditWriter` → `ActorAuditWriter`, whose ctor calls `registry.Get()` — and `WithSingleton` is registered ~60 lines later in the same builder. Same class as scadaproj#1 and mesh Phase 6's `IClusterRoleInfo` crash: only visible with the DI graph built in a real host, which is why the container-activation test (no `AddZbSecrets` in its collection) and `TwoNodeClusterHarness` (no provider at all) both miss it. Aggravating: the failed process does not exit, so `docker ps` reports `Up restarts=0` while the node serves no HTTP | **OPEN.** Fix + red-first pin belong to 1A (the gate made no product change). Suggested shape: hand the actor lazy accessors, mirroring the `Func<>` the same block already uses for the coordinator proxy. Pin must build the graph in starter order (a Host.IntegrationTests boot of an admin-role host), not merely activate the dialer | | 3 | 1A.2 | **Fan parked the actor mailbox** (self-caught by the implementer): dialing from the receive ran `TryGetPsk`'s cold-cache 3 s block pre-await (cold after EVERY refresh due to `InvalidateAll`), stalling deployment acks behind the fan | async methods run synchronously to the first await | `Task.Run` hop (token None — a canceled scheduling would read as a DB failure via PipeTo); pinned red-first by `A_slow_dial_does_not_park_the_actors_mailbox` | ## Carry-forward register @@ -140,15 +141,18 @@ Checks a gate could not drive, named + carried to the next gate (proportionate-g | Hang-style central failover drill (`docker pause`, not `docker stop` — only a hang discriminates the per-endpoint-deadline fix) | 1B.2 review | unit fact `Each_endpoint_gets_a_FRESH_deadline` | 1B.3 gate (MUST include) | | I-4 measurement: ack burst within ~3 s after a contact-refresh tick (cold PSK cache on the inbound auth path) | 1A.2 review | bounded 3 s store read + `Lazy` stampede suppression | 1B.3 gate (observe) | | `TwoNodeClusterHarness` logs the I-1 null-provider Error at every spawn (harness registers no `IMeshPskProvider`) — EXPECTED, not a rig fault | 1A.2 re-review | production pairing is `hasAdmin`-unconditional in one block | note for 1B.3/Phase 2 readers | -| Register item 3 (DI role-scoping pin: admin resolves provider+dialer, driver-only does NOT) — container-activation test exists, role-scoping pin outstanding | 1A.2 | `It_activates_from_the_container_despite_the_optional_client_factory_parameter` | Phase 2 (or 1B.3) | +| Register item 3 (DI role-scoping pin: admin resolves provider+dialer, driver-only does NOT) — container-activation test exists, role-scoping pin outstanding | 1A.2 | `It_activates_from_the_container_despite_the_optional_client_factory_parameter` | Phase 2 (or 1B.3) — note defect 4 shows the missing pin is not merely role-scoping but **starter-order**: the container-activation test resolves from a collection without `AddZbSecrets` | +| 1A checks 1/2/7 (deploy fan over gRPC, mixed-transport seal, NULL-`GrpcPort` skip) and the central halves of 3/4 | 1A.3 gate | Nothing live — unit coverage only | **1A gate re-run after defect 4**; must not be carried past 1A | +| Check 8 is vacuous for telemetry + config-serve — central never booted, so neither plane carried a call | 1A.3 gate | LocalDb sync did carry traffic, 0 denials | 1A gate re-run | +| **`DriverStatusPanel.razor` is referenced by no page or component** — the Reconnect/Restart buttons are unreachable in the running AdminUI (pre-existing on `master`, not caused by this program) | 1A.3 gate | Node-side leg driven with real envelopes | Out of program scope — file separately if the control is meant to exist | ## Issues filed by this program | Issue | What | Status | |---|---|---| | [lmxopcua#542](https://gitea.dohertylan.com/dohertj2/lmxopcua/issues/542) | `FleetStatusBroadcaster` derives "fleet" from central's own gossip ring — silently central-pair-blind since mesh Phase 6; pre-existing, transport-independent, out of this program's scope | FILED 2026-08-06 | -| *(to file in 1A.3)* | `ConfigPublishCoordinatorTimeoutTests.DeadlineElapsed_for_current_deployment_marks_TimedOut` is racy on its own terms (150 ms wall-clock deadline; row inserted after spawn) — flaked 2/~11 full runs once 1A.2 grew the suite; green at base. Fix the test (inject clock / raise deadline) | to file | -| *(to file in 1A.3)* | Mesh command/ack planes are h2c-only — the per-cluster PSK crosses the network in cleartext on a write surface (precedent: every sibling gRPC plane; trusted-site-link assumption). Decide: accepted risk, or a `UseTls`/scheme option | to file | +| [lmxopcua#544](https://gitea.dohertylan.com/dohertj2/lmxopcua/issues/544) | `ConfigPublishCoordinatorTimeoutTests.DeadlineElapsed_for_current_deployment_marks_TimedOut` is racy on its own terms (150 ms wall-clock deadline; row inserted after spawn) — flaked 2/~11 full runs once 1A.2 grew the suite; green at base. Fix the test (inject clock / raise deadline) | FILED 2026-08-06 (1A.3) | +| [lmxopcua#545](https://gitea.dohertylan.com/dohertj2/lmxopcua/issues/545) | Mesh command/ack planes are h2c-only — the per-cluster PSK crosses the network in cleartext on a write surface (precedent: every sibling gRPC plane; trusted-site-link assumption). Decide: accepted risk, or a `UseTls`/scheme option | FILED 2026-08-06 (1A.3) | | [lmxopcua#543](https://gitea.dohertylan.com/dohertj2/lmxopcua/issues/543) | The three pre-existing auth interceptors (`TelemetryStream`/`ConfigServe`/`LocalDbSync`) treat a whitespace-only key as SET (`IsNullOrEmpty`), so a whitespace `Telemetry:ApiKey` etc. opens the endpoint to a whitespace bearer; the two new mesh interceptors use `IsNullOrWhiteSpace`. Backport is a behaviour change to shipped endpoints — own issue, out of this program's scope | FILED 2026-08-06 | ## Known constraints carried from the research (do not rediscover) From c0475f2df49d6a484e28ee9ebd429e88373ec1ea Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 6 Aug 2026 07:55:31 -0400 Subject: [PATCH 09/12] =?UTF-8?q?fix(mesh):=20resolve=20dialer=20+=20PSK?= =?UTF-8?q?=20provider=20lazily=20=E2=80=94=20central=20boots=20again?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1A live gate defect 4. Neither central node booted on this branch, in ANY transport mode: Akka.Hosting.MissingActorRegistryEntryException: No actor registered for key ...ControlPlane.AuditWriterActorKey at AdminUI.Audit.ActorAuditWriter..ctor(ActorRegistry registry) at ControlPlane.ServiceCollectionExtensions. WithOtOpcUaControlPlaneSingletons' first WithActors starter resolved IMeshCommandDialer and IMeshPskProvider eagerly for the Props. Their graph reaches back into the registry — dialer -> IMeshPskProvider -> ISecretResolver -> AddZbSecrets -> IAuditWriter -> ActorAuditWriter, whose ctor calls registry.Get() — and WithSingleton is registered ~60 lines LATER in the same builder. Boot died before the entry existed. Mode-independent: the resolve happened before anything consulted MeshTransport:Mode, so Dps (the default, and the rollback target) died exactly like Grpc. Both dependencies now arrive as Func<> and are invoked in PreStart, by which point the host is fully built — the same shape, for the same class of reason, as the coordinator proxy Func one argument earlier. PreStart re-resolves on restart rather than caching across one; both are container singletons, so a restart gets the same instances. A resolution throw is caught and logged rather than propagated: a throw out of PreStart is an actor failure, the supervisor answers it with a restart, and a broken DI graph is not transient — that is an unbounded restart loop that would take the transport-independent ack path down with it. Regression pin: ControlPlaneStarterOrderTests boots a REAL admin host through WithOtOpcUaClusterBootstrap + WithOtOpcUaControlPlaneSingletons in production starter order, with the production graph from IMeshCommandDialer down to a registry-bound IAuditWriter, and asserts StartAsync itself — the failure occurs while the builder's starters execute, so building a ServiceProvider and resolving by hand cannot reproduce it. Verified RED against the eager resolve: 3/3 modes fail with MissingActorRegistryEntryException. A second fact pins that the deferral did not become 'never resolved' — the actor must resolve the provider from the container and drive it through the contact refresh, signalled by a TaskCompletionSource (PreStart runs on the actor thread and the contact load is a piped DB read, so no synchronous assertion after StartAsync is correct). Why nothing caught this: the dialer's container-activation test resolves from a collection with no secrets registration, so its graph stops one hop in; the actor facts inject fakes and never touch DI; and the existing whole-host boots register no mesh provider, so the eager lookups returned null and walked nothing. Two aggravators recorded by the gate, no code here: the failed process did not exit (docker Up while /health/ready 503s — Phase 2 readiness concern, tracked), and this is the third instance of the same eager-resolve-during-AddAkka class after scadaproj#1 and mesh Phase 6's IClusterRoleInfo boot crash. ControlPlane.Tests 236 -> 240, Host.Tests 92, Cluster.Tests 184, 0 warnings in every touched project. --- .../CentralCommunicationActor.cs | 103 ++++++-- .../ServiceCollectionExtensions.cs | 14 +- .../CentralCommunicationActorTests.cs | 8 +- .../ControlPlaneStarterOrderTests.cs | 233 ++++++++++++++++++ 4 files changed, 333 insertions(+), 25 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/ControlPlaneStarterOrderTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs index 60f3b7c2..79166703 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs @@ -64,8 +64,15 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers private readonly MeshTransportOptions _options; private readonly IMeshClusterClientFactory _clientFactory; private readonly Func _coordinator; - private readonly IMeshCommandDialer? _dialer; - private readonly IMeshPskProvider? _pskProvider; + + // Resolved in PreStart, never at construction. See the Props doc: the DI graph behind these two + // reaches actor-registry entries that do not exist yet while the Akka builder's starters are + // still running, and touching it there is what stopped every central node from booting. + private readonly Func? _dialerFactory; + private readonly Func? _pskProviderFactory; + private IMeshCommandDialer? _dialer; + private IMeshPskProvider? _pskProvider; + private readonly ILoggingAdapter _log = Context.GetLogger(); private readonly bool _useClusterClient; private readonly bool _useGrpc; @@ -100,24 +107,45 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers /// time — it resolves on first use and caches. /// /// - /// Places one command call on one node, under . - /// Optional so the two older transports need nothing new wired; a - /// dialer under Grpc is a mis-wiring and is reported loudly rather than silently - /// dropping every command. + /// Resolves the thing that places one command call on one node, under + /// . Optional so the two older transports need + /// nothing new wired; a dialer under Grpc is a mis-wiring and + /// is reported loudly rather than silently dropping every command. /// /// - /// Central's per-cluster key lookup, told on every contact refresh which clusters exist (the - /// known-cluster gate) and to drop its cache (so a rotated key takes effect within one - /// refresh). Optional for the same reason as . + /// Resolves central's per-cluster key lookup, told on every contact refresh which clusters + /// exist (the known-cluster gate) and to drop its cache (so a rotated key takes effect within + /// one refresh). Optional for the same reason as . /// /// The props. + /// + /// + /// Both dependencies arrive as , and that is not a style + /// choice. Passing instances meant the caller resolved them while the Akka builder's + /// starters were still running, and the DI graph behind them reaches back into the actor + /// registry: dialer → IMeshPskProvider → ISecretResolver → AddZbSecrets + /// → IAuditWriter → ActorAuditWriter, whose constructor calls + /// registry.Get<AuditWriterActorKey>() — a singleton registered dozens of + /// lines LATER in the very same builder. Every central node failed to boot, in every + /// transport mode, on a MissingActorRegistryEntryException. Deferring to + /// moves the first resolve to a point where the host is fully + /// built, and matches the the coordinator proxy already uses + /// one argument earlier for the same class of reason. + /// + /// + /// This is the third instance of the same failure mode in this family — eager resolution + /// inside an AddAkka configurator, previously scadaproj#1 (a DI cycle that + /// deadlocked) and mesh Phase 6's IClusterRoleInfo boot StackOverflow. The + /// rule it keeps re-teaching: inside an Akka builder, resolve nothing you can defer. + /// + /// public static Props Props( IDbContextFactory dbFactory, MeshTransportOptions options, IMeshClusterClientFactory clientFactory, Func coordinator, - IMeshCommandDialer? dialer = null, - IMeshPskProvider? pskProvider = null) => + Func? dialer = null, + Func? pskProvider = null) => Akka.Actor.Props.Create(() => new CentralCommunicationActor( dbFactory, options, clientFactory, coordinator, dialer, pskProvider)); @@ -127,22 +155,22 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers /// The bound mesh-transport options. /// Creates the ClusterClient. /// Lazily resolves the deploy-coordinator singleton proxy. - /// Places one gRPC command call on one node; Grpc mode only. - /// Central's per-cluster key lookup; Grpc mode only. + /// Resolves the gRPC command dialer; invoked in . + /// Resolves the PSK provider; invoked in . public CentralCommunicationActor( IDbContextFactory dbFactory, MeshTransportOptions options, IMeshClusterClientFactory clientFactory, Func coordinator, - IMeshCommandDialer? dialer = null, - IMeshPskProvider? pskProvider = null) + Func? dialer = null, + Func? pskProvider = null) { _dbFactory = dbFactory; _options = options; _clientFactory = clientFactory; _coordinator = coordinator; - _dialer = dialer; - _pskProvider = pskProvider; + _dialerFactory = dialer; + _pskProviderFactory = pskProvider; _useClusterClient = string.Equals( options.Mode, MeshTransportOptions.ModeClusterClient, StringComparison.OrdinalIgnoreCase); _useGrpc = string.Equals( @@ -183,6 +211,12 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers /// protected override void PreStart() { + // First point at which the host is fully built — see the Props remark. Re-runs on every + // actor restart, which re-reads the container rather than caching a stale reference; both + // dependencies are container singletons, so a restart gets the same instances back. + _dialer = Resolve(_dialerFactory, nameof(IMeshCommandDialer)); + _pskProvider = Resolve(_pskProviderFactory, nameof(IMeshPskProvider)); + if (!_useClusterClient && !_useGrpc) { _log.Info( @@ -229,6 +263,41 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers TimeSpan.FromSeconds(_options.ContactRefreshSeconds)); } + /// + /// Invokes a dependency factory, converting a resolution failure into a logged absence. + /// + /// The dependency's type. + /// The factory, or when nothing was supplied. + /// The dependency's name, for the log. + /// The resolved dependency, or . + /// + /// Catching is the point. A throw out of is an actor failure, and the + /// supervisor's answer to that is to restart — which calls again, and + /// a broken DI graph is not a transient condition. That is an unbounded restart loop on the + /// node's command boundary, taking the transport-independent ack path down with it. Logging + /// and continuing leaves the ack path serving and hands the two null guards below a genuine + /// absence to report, which is the same outcome as never having been wired. + /// + private T? Resolve(Func? factory, string name) + where T : class + { + if (factory is null) + return null; + + try + { + return factory(); + } + catch (Exception ex) + { + _log.Error( + ex, + "Failed to resolve {Dependency} for the mesh command boundary; continuing without " + + "it. The guards below report what that costs", name); + return null; + } + } + /// protected override void PostStop() { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs index 32a04231..1801da38 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs @@ -66,14 +66,20 @@ public static class ServiceCollectionExtensions var dbFactory = resolver.GetService>(); var options = resolver.GetService>().Value; + // Captured, NOT resolved. Resolving the mesh dialer or PSK provider here made every + // central node fail to boot in every transport mode: their graph is dialer → + // IMeshPskProvider → ISecretResolver → AddZbSecrets → IAuditWriter → ActorAuditWriter, + // whose constructor calls registry.Get() — and WithSingleton + // is registered further down THIS builder, so the entry does not + // exist while this starter runs. The actor invokes these in PreStart instead, by which + // point the host is built. Same shape, same reason, as the coordinator Func below. + // // Through IServiceProvider rather than the Akka resolver, deliberately: these two are // OPTIONAL (Grpc mode only, and admin-role only within that), and GetService's // null-for-absent contract is the thing being relied on. The Akka resolver's // GetService makes no such promise, and this same block runs in test harnesses whose // service collections register neither. var serviceProvider = resolver.GetService(); - var dialer = serviceProvider.GetService(); - var pskProvider = serviceProvider.GetService(); var actor = system.ActorOf( CentralCommunicationActor.Props( @@ -85,8 +91,8 @@ public static class ServiceCollectionExtensions () => registry.TryGet(out var coordinator) ? coordinator : null, - dialer, - pskProvider), + () => serviceProvider.GetService(), + () => serviceProvider.GetService()), MeshPaths.CentralCommunicationName); // Registered unconditionally, in both transport modes. Under Dps nothing dials it, and diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs index 5556295d..9004bf2d 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs @@ -742,8 +742,8 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase new MeshTransportOptions { Mode = mode, ContactRefreshSeconds = 60 }, clients, () => null, - new RecordingDialer(), - psk)); + () => new RecordingDialer(), + () => psk)); AwaitCondition( () => psk.KnownClusterPublications.Count >= 1, TimeSpan.FromSeconds(5)); @@ -855,8 +855,8 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase GrpcMode(), clientFactory ?? new RecordingClientFactory(() => CreateTestProbe()), () => coordinator, - dialer, - psk)); + () => dialer, + () => psk)); return new GrpcRig(actor, dialer, psk); } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/ControlPlaneStarterOrderTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/ControlPlaneStarterOrderTests.cs new file mode 100644 index 00000000..589c34f2 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/ControlPlaneStarterOrderTests.cs @@ -0,0 +1,233 @@ +using System.Diagnostics.CodeAnalysis; +using Akka.Actor; +using Akka.Cluster.Hosting; +using Akka.Hosting; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.Audit; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; +using ZB.MOM.WW.Secrets.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication; + +/// +/// Guards the one thing every other test in this project structurally cannot see: whether an +/// admin node boots with the real registration graph behind the mesh command boundary. +/// +/// +/// +/// The Phase 1A live gate failed here. WithOtOpcUaControlPlaneSingletons's first +/// starter resolved IMeshCommandDialer and IMeshPskProvider eagerly, and their +/// graph reaches back into the actor registry — dialer → IMeshPskProvider → +/// ISecretResolver → AddZbSecrets → IAuditWriter → +/// ActorAuditWriter, whose constructor calls +/// registry.Get<AuditWriterActorKey>() for a singleton registered further down +/// the SAME builder. Every central node died on +/// MissingActorRegistryEntryException, in every transport mode, before serving +/// anything. +/// +/// +/// Why the existing tests were blind. MeshGrpcCommandDialerTests activates the +/// dialer from a ServiceCollection that has no secrets registration at all, so its +/// graph stops one hop in; the CentralCommunicationActorTests facts inject fakes +/// directly and never touch DI; and the existing whole-host boots register no mesh provider, +/// so the eager lookups returned and resolved nothing. Every one of +/// them passed while no central node could start. +/// +/// +/// So the assertion is StartAsync itself. The failure happens while the Akka +/// builder's starters execute, which is inside StartAsync — building the +/// ServiceProvider and resolving from it by hand would not reproduce it. The graph +/// below is the production one from IMeshCommandDialer down to the registry-bound +/// audit writer; only the secret store itself is a stand-in, because a real one would add a +/// KEK and a SQLite file to a test whose subject is starter ORDER. +/// +/// +public sealed class ControlPlaneStarterOrderTests +{ + [Theory] + [InlineData(MeshTransportOptions.ModeDps)] + [InlineData(MeshTransportOptions.ModeClusterClient)] + [InlineData(MeshTransportOptions.ModeGrpc)] + public async Task An_admin_node_boots_with_the_real_mesh_dependency_graph(string mode) + { + // Every mode, because the eager resolve was mode-independent: the starter resolved both + // dependencies before anything looked at MeshTransport:Mode, so Dps — the DEFAULT, and the + // rollback target — died exactly like Grpc did. + using var host = BuildAdminHost(mode); + + await Should.NotThrowAsync(() => host.StartAsync()); + + try + { + var registry = host.Services.GetRequiredService(); + registry.TryGet(out _).ShouldBeTrue( + "the command boundary must be registered on a booted admin node"); + registry.TryGet(out _).ShouldBeTrue( + "the audit singleton the eager resolve reached for too early must exist by now"); + } + finally + { + await host.StopAsync(); + } + } + + [Fact] + public async Task The_deferred_dependencies_are_resolved_and_used_once_the_host_is_up() + { + // Deferring must not become "never resolved" — that would trade a loud boot failure for a + // silent one, with the known-cluster gate permanently unpublished and the dialer permanently + // absent. So this asserts the far end of the deferral: the actor resolved the PSK provider + // FROM THE CONTAINER and drove it, through the ordinary contact-refresh path. + // + // The signal is a TaskCompletionSource rather than a counter, because PreStart runs on the + // actor's own thread and the contact load is a piped DB read: there is no instant after + // StartAsync at which a synchronous assertion is correct. + var signal = new SignallingPskProvider(); + using var host = BuildAdminHost(MeshTransportOptions.ModeGrpc, signal); + + await host.StartAsync(); + + try + { + await signal.Published.WaitAsync(TimeSpan.FromSeconds(20)); + } + finally + { + await host.StopAsync(); + } + } + + /// + /// Builds an admin host through the production cluster bootstrap and the production control-plane + /// starter registration, with the mesh dependency graph wired as it is on a real central node. + /// + /// The mesh transport mode to boot under. + /// + /// Substitute for the real . Only the resolution test supplies + /// one; the boot test deliberately keeps the production type so the graph it walks is the + /// production graph. + /// + /// The unstarted host. + private static IHost BuildAdminHost(string mode, IMeshPskProvider? pskProvider = null) + { + var clusterOptions = new AkkaClusterOptions + { + Port = 0, + Hostname = "127.0.0.1", + PublicHostname = "127.0.0.1", + SeedNodes = [], + Roles = ["admin"], + }; + + return Host.CreateDefaultBuilder() + .ConfigureServices(services => + { + services.AddSingleton>(Options.Create(clusterOptions)); + services.AddSingleton>(Options.Create( + new MeshTransportOptions { Mode = mode, DispatchTimeoutSeconds = 10 })); + services.AddDbContextFactory( + o => o.UseInMemoryDatabase($"starter-order-{Guid.NewGuid():N}")); + services.AddSingleton(new StubClusterRoleInfo()); + + // THE GRAPH. Production types from the dialer down to the registry-bound writer; the + // order of these registrations is deliberately irrelevant, because the defect was + // never about registration order — it was about WHEN the graph is first walked. + services.AddSingleton(); + services.AddSingleton(); + if (pskProvider is null) + services.AddSingleton(); + else + services.AddSingleton(pskProvider); + services.AddSingleton(); + + services.AddAkka("otopcua-starter-order-test", (ab, sp) => + { + ab.WithOtOpcUaClusterBootstrap(sp); + ab.WithOtOpcUaControlPlaneSingletons(); + }); + }) + .Build(); + } + + /// + /// Stands in for ActorAuditWriter: an that resolves a + /// cluster singleton from the in its constructor. That + /// constructor is the tripwire — it throws + /// MissingActorRegistryEntryException for exactly as long as the audit singleton has + /// not been registered yet. + /// + private sealed class RegistryBoundAuditWriter : IAuditWriter + { + private readonly IActorRef _proxy; + + public RegistryBoundAuditWriter(ActorRegistry registry) => _proxy = registry.Get(); + + public Task WriteAsync(AuditEvent evt, CancellationToken ct = default) + { + _proxy.Tell(evt); + return Task.CompletedTask; + } + } + + /// + /// An that completes a task the first time the actor publishes + /// the known-cluster set. Takes it never uses, purely so that + /// resolving it still walks the same registry-bound chain the real provider does. + /// + private sealed class SignallingPskProvider : IMeshPskProvider + { + private readonly TaskCompletionSource _published = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// Completes once has been called. + public Task Published => _published.Task; + + public bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk) + { + psk = null; + return false; + } + + public bool Matches(string clusterId, string presented) => false; + + public void SetKnownClusters(IReadOnlyCollection clusterIds) => _published.TrySetResult(); + + public void Invalidate(string clusterId) { } + + public void InvalidateAll() { } + } + + /// + /// Stands in for the shared secret store's resolver. Its only load-bearing property is the + /// one that broke boot: it depends on , exactly as + /// AddZbSecrets' resolver registration does. + /// + private sealed class AuditingSecretResolver : ISecretResolver + { + public AuditingSecretResolver(IAuditWriter auditWriter) => ArgumentNullException.ThrowIfNull(auditWriter); + + public Task GetAsync(SecretName name, CancellationToken ct) => Task.FromResult(null); + } + + /// Minimal ; the admin singleton factories read it. + private sealed class StubClusterRoleInfo : IClusterRoleInfo + { + public string? ClusterRole => null; + public string? ClusterId => null; + public NodeId LocalNode => NodeId.Parse("127.0.0.1:0"); + public IReadOnlySet LocalRoles => new HashSet(); + public bool HasRole(string role) => false; + public IReadOnlyList MembersWithRole(string role) => []; + public NodeId? RoleLeader(string role) => null; + public event EventHandler? RoleLeaderChanged { add { } remove { } } + } +} From e54883a8e049dcd96b9e73a798447f6f1945de35 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 6 Aug 2026 08:07:34 -0400 Subject: [PATCH 10/12] =?UTF-8?q?fix(mesh):=20memoize=20dialer/provider=20?= =?UTF-8?q?on=20first=20use=20=E2=80=94=20a=20lost=20PreStart=20race=20sel?= =?UTF-8?q?f-heals=20at=20the=20first=20refresh=20(review-prescribed)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../CentralCommunicationActor.cs | 18 ++++++-- .../ControlPlaneStarterOrderTests.cs | 42 +++++++++++++------ 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs index 79166703..e25342f4 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs @@ -73,6 +73,18 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers private IMeshCommandDialer? _dialer; private IMeshPskProvider? _pskProvider; + // Memoize-on-first-use, NOT PreStart-only: PreStart runs on a dispatcher thread concurrently + // with the builder's remaining starters, so its resolve can lose a millisecond-wide race + // against AuditWriterActorKey's registration (starter #4) and Resolve() would swallow that + // into a PERMANENT silent null — the known-cluster gate never published, fail-open. These + // properties retry on first use instead; the first RefreshContacts arrives as a message, + // long after every starter finished, so a lost boot race self-heals there. Actor-thread + // only, so ??= needs no synchronisation; both are container singletons, so a repeat resolve + // is a dictionary hit. + private IMeshPskProvider? PskProvider => _pskProvider ??= Resolve(_pskProviderFactory, nameof(IMeshPskProvider)); + + private IMeshCommandDialer? Dialer => _dialer ??= Resolve(_dialerFactory, nameof(IMeshCommandDialer)); + private readonly ILoggingAdapter _log = Context.GetLogger(); private readonly bool _useClusterClient; private readonly bool _useGrpc; @@ -382,7 +394,7 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers { var commandType = cmd.Message.GetType().Name; - var dialer = _dialer; + var dialer = Dialer; if (dialer is null) { _log.Error( @@ -796,8 +808,8 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers /// private void PublishKnownClusters(ContactsLoaded msg) { - _pskProvider?.SetKnownClusters(msg.ClusterIds); - _pskProvider?.InvalidateAll(); + PskProvider?.SetKnownClusters(msg.ClusterIds); + PskProvider?.InvalidateAll(); } /// Stops a cluster's existing client and creates a replacement for its new contact set. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/ControlPlaneStarterOrderTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/ControlPlaneStarterOrderTests.cs index 589c34f2..6c90b7c9 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/ControlPlaneStarterOrderTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/ControlPlaneStarterOrderTests.cs @@ -80,24 +80,35 @@ public sealed class ControlPlaneStarterOrderTests } } - [Fact] - public async Task The_deferred_dependencies_are_resolved_and_used_once_the_host_is_up() + [Theory] + [InlineData(MeshTransportOptions.ModeDps)] + [InlineData(MeshTransportOptions.ModeClusterClient)] + [InlineData(MeshTransportOptions.ModeGrpc)] + public async Task The_deferred_dependencies_are_resolved_and_used_once_the_host_is_up(string mode) { // Deferring must not become "never resolved" — that would trade a loud boot failure for a // silent one, with the known-cluster gate permanently unpublished and the dialer permanently // absent. So this asserts the far end of the deferral: the actor resolved the PSK provider - // FROM THE CONTAINER and drove it, through the ordinary contact-refresh path. + // FROM THE CONTAINER and drove it, through the ordinary contact-refresh path. A Theory over + // all three modes because publication is mode-independent (the gate follows the endpoint). + // + // The provider is registered as a TYPE with an ISecretResolver constructor, so the resolve + // walks the same registry-bound chain the production MeshPskProvider does — which makes + // this fact the race detector for the PreStart-vs-starter-#4 window: a lost race that + // stayed null forever would never publish, and the TaskCompletionSource below would time + // out red. (Memoize-on-first-use in the actor is what heals the lost race at the first + // RefreshContacts message.) // // The signal is a TaskCompletionSource rather than a counter, because PreStart runs on the // actor's own thread and the contact load is a piped DB read: there is no instant after // StartAsync at which a synchronous assertion is correct. - var signal = new SignallingPskProvider(); - using var host = BuildAdminHost(MeshTransportOptions.ModeGrpc, signal); + using var host = BuildAdminHost(mode, useSignallingProvider: true); await host.StartAsync(); try { + var signal = (SignallingPskProvider)host.Services.GetRequiredService(); await signal.Published.WaitAsync(TimeSpan.FromSeconds(20)); } finally @@ -111,13 +122,15 @@ public sealed class ControlPlaneStarterOrderTests /// starter registration, with the mesh dependency graph wired as it is on a real central node. /// /// The mesh transport mode to boot under. - /// - /// Substitute for the real . Only the resolution test supplies - /// one; the boot test deliberately keeps the production type so the graph it walks is the - /// production graph. + /// + /// Substitutes for the real . + /// Only the resolution test sets it; the boot test keeps the production type so the graph it + /// walks is the production graph. The signalling provider is registered as a factory taking + /// from the container, so its resolution walks the same + /// registry-bound chain either way. /// /// The unstarted host. - private static IHost BuildAdminHost(string mode, IMeshPskProvider? pskProvider = null) + private static IHost BuildAdminHost(string mode, bool useSignallingProvider = false) { var clusterOptions = new AkkaClusterOptions { @@ -143,10 +156,11 @@ public sealed class ControlPlaneStarterOrderTests // never about registration order — it was about WHEN the graph is first walked. services.AddSingleton(); services.AddSingleton(); - if (pskProvider is null) - services.AddSingleton(); + if (useSignallingProvider) + services.AddSingleton(sp => + new SignallingPskProvider(sp.GetRequiredService())); else - services.AddSingleton(pskProvider); + services.AddSingleton(); services.AddSingleton(); services.AddAkka("otopcua-starter-order-test", (ab, sp) => @@ -188,6 +202,8 @@ public sealed class ControlPlaneStarterOrderTests private readonly TaskCompletionSource _published = new(TaskCreationOptions.RunContinuationsAsynchronously); + public SignallingPskProvider(ISecretResolver resolver) => ArgumentNullException.ThrowIfNull(resolver); + /// Completes once has been called. public Task Published => _published.Task; From e3a0ddbc84076c076ff0413ed37ebcf07803340e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 6 Aug 2026 08:37:48 -0400 Subject: [PATCH 11/12] =?UTF-8?q?docs(mesh):=20Phase=201A=20live=20gate=20?= =?UTF-8?q?re-run=20=E2=80=94=20PASS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-ran the 1A gate on the docker-dev rig at e54883a8 after the defect-4 boot fix. Image rebuilt (otopcua-host:dev -> 1d8452a29018); six host nodes plus both one-shots recreated; run 1's preserved image was stale and deliberately not reused. VERDICT: PASS. Boot: 6/6 nodes healthy INCLUDING the central pair, 0 Hosting failed to start, /health/ready 200 on every node, MAIN mesh 2/2. Check 1 PASS: one DispatchDeployment, one correlation id, accepted by all six nodes; central logs 6 accepted / 0 failed; zero ClusterClient lines on either central node with Akka.Cluster.Tools.Client at Debug. Check 2 PASS: deployment Sealed with all six NodeDeploymentState rows Applied inside ~90 ms, while the four site nodes hold receptionist associations — the only ack path they have in 1A. Mixed transport proven. Check 4 PASS: Ack clicked on the real /alerts page against a live ACTIVATED row; central fanned one AlarmCommand to all six over gRPC and the condition acknowledged 0.54 s later carrying the operator identity. Check 7 PASS: with site-b-2's GrpcPort NULLed, both central nodes warned once and the next dispatch reached exactly five nodes; the deployment then honestly ended TimedOut with site-b-2 alone unapplied. Row restored, 6/6 sealed after. Check 8 PASS and no longer vacuous: 0 denials on all four interceptors, and this time config-serve (28 KB verified artifact fetches from :4055) and telemetry (streams open on all six, live driver health on /hosts) both carried real traffic. Check 3 closed by composition, with the reason now established rather than assumed: DispatchOverGrpc origin path from the AdminUI. Checks 5 + 6 stand from run 1 — nothing in the fix touches the interceptor. New finding 5, recorded because it is load-bearing for the fix's design: the PreStart resolve LOSES its race on real boots — 2 of 3 observed admin boots logged Failed to resolve for both dependencies with the same AuditWriterActorKey cause. The memoize-on-first-use hardening is therefore not theoretical, and its heal was proven live by moving the coordinator singleton onto the node that lost the race (stop central-2) and watching central-1 fan a dispatch to all five live nodes with no dialer error. Residual is cosmetic and carried to Phase 2: the Error text reads as permanent, and it will trip a soak error-grep on a healthy node. Defect 4 marked CLOSED; run 1's record kept in full underneath the combined table. Rig restored: flip reverted, six nodes healthy on all-ClusterClient on the committed image, three mesh ClusterClients back on each central node. --- ...6-08-06-clusterclient-to-grpc-live-gate.md | 357 ++++++++++++++++-- ...26-08-06-clusterclient-to-grpc-tracking.md | 12 +- 2 files changed, 327 insertions(+), 42 deletions(-) diff --git a/docs/plans/2026-08-06-clusterclient-to-grpc-live-gate.md b/docs/plans/2026-08-06-clusterclient-to-grpc-live-gate.md index d1811c24..e18b860b 100644 --- a/docs/plans/2026-08-06-clusterclient-to-grpc-live-gate.md +++ b/docs/plans/2026-08-06-clusterclient-to-grpc-live-gate.md @@ -154,7 +154,270 @@ Rig left running on `ClusterClient` (its normal state) after the gate. --- -## Phase 1A — central→node command plane — **FAIL (blocked at the central boot boundary)** +## Phase 1A — central→node command plane — **PASS (on the re-run at `e54883a8`)** + +Two runs. **Run 1 @ `fbf7b708` FAILED** at the central boot boundary (defect 4) and could not +drive checks 1/2/7 or the central halves of 3/4; its checks 5 + 6 and the node-side halves of +3 + 4 stand as recorded and were not repeated. **Run 2 @ `e54883a8`** — after the boot fix +(`c0475f2d`) and its review-hardening follow-up (`e54883a8`) — drove every carried check. The +result table below is the **combined, current** verdict; run 1's failure record is preserved +underneath it because the defect it caught is the most valuable thing this gate produced. + +| # | Check | Verdict | Run | +|---|---|---|---| +| — | **Boot:** all six nodes healthy INCLUDING the central pair, on the branch image | **PASS** — 6/6 `running restarts=0`, 0 `Hosting failed to start`, `/health/ready` 200 ×6, MAIN mesh `memberCount=2 unreachableCount=0`. **But the `Resolve` Error line the fix logs DID fire — see finding 5** | 2 | +| 1 | Deploy dispatch reaches **all six** driver nodes over gRPC; zero `SendToAll`/ClusterClient on central | **PASS** — 6/6 nodes accepted one `DispatchDeployment` under a single shared `correlationId=cf499b42…`; central's own per-node outcome log records 6 accepted / 0 failed; `grep -ci clusterclient` = **0** on both central nodes | 2 | +| 2 | Acks return over ClusterClient (site mode unchanged) and the deployment **seals** | **PASS** — deployment `8DF9AAE0…` `Status=2` (Sealed) with all six `NodeDeploymentState` rows `Status=1` (Applied) inside ~90 ms; all four site nodes hold live `Connected to [akka.tcp://…/system/receptionist]` associations, which is their only ack path in 1A | 2 | +| 3 | `RestartDriver` + `ReconnectDriver` land on the target site node | **PASS (composed)** — node-side leg driven live in run 1 (both commands acted on site-a-1). The AdminUI half **cannot be driven and this is now proven, not assumed**: ` 70000`), so a transition was manufactured by +editing its predicate to an always-true form and deploying; the `/alerts` page was already open +when it fired, the Ack button was clicked, and the predicate + hash were restored afterwards +(verified byte-identical by `SELECT`) with a further deploy: + +``` +page open at 2026-08-06T12:20:59.036Z initial: empty window +ROWS: 12:21:00.017 gate481-alarm | PressHigh | EQ-6e09bcc2d907 | ACTIVATED | 750 | system | press high +ROW BUTTONS: ["Ack","Shelve","Shelve (timed)","Unshelve"] +clicking Acknowledge at 2026-08-06T12:21:00.362Z +after click: 12:21:00.557 | gate481-alarm | EQ-6e09bcc2d907 | ACKNOWLEDGED | 750 | multi-role-test | press high + +$ docker logs otopcua-dev-central-2-1 | grep AlarmCommand +[12:21:00 DBG] Mesh: AlarmCommand (correlationId=ccafe3754aca40cfa8aa0a0602c86cbb) accepted by site-b-2:4053 over gRPC +[12:21:00 DBG] … accepted by central-1:4053 … [12:21:00 DBG] … accepted by central-2:4053 … +[12:21:00 DBG] … accepted by site-a-1:4053 … [12:21:00 DBG] … accepted by site-b-1:4053 … +[12:21:00 DBG] … accepted by site-a-2:4053 … +$ for c in …; do grep "Mesh: accepted AlarmCommand" …; done → all six, same correlationId +``` + +One correlation id across all six recipients is the "encode once, before the fan" rule holding +for the one command whose CLR record has no correlation member of its own — the case the design +called out as the reason encoding is hoisted out of the loop. + +**Check 7 — the skip, the warn, and the honest deadline.** + +``` +$ … UPDATE dbo.ClusterNode SET GrpcPort=NULL WHERE NodeId='site-b-2:4053' # 12:23:14 + +# one warn per central node, once each, within one ContactRefreshSeconds: +central-2 [12:23:20 WRN] ClusterNode site-b-2:4053 has no GrpcPort; it exposes no command endpoint + and is skipped in this dispatch set (further skips of this node are silent). Commands + will not reach it until the row carries a port +central-1 [12:23:19 WRN] …identical… +# (each node also logs the TelemetryNodeSource twin — "exposes no telemetry stream … skipped in +# this dial refresh" — expected collateral: both target sets read the same column) + +# dispatch at 12:23:36 — accepted by five, site-b-2 absent: +central-2 [12:23:37 DBG] … (correlationId=88a9e0a2…) accepted by central-2 / site-a-2 / site-b-1 + / site-a-1 / central-1 :4053 over gRPC +# cumulative per-node accept counts after this dispatch — site-b-2 one behind everyone: +central-1 4 central-2 4 site-a-1 4 site-a-2 4 site-b-1 4 site-b-2 3 + +$ … SELECT Status FROM dbo.Deployment WHERE DeploymentId='5c8bdc9e-…' → 4 (TimedOut) +$ … SELECT NodeId, Status FROM dbo.NodeDeploymentState … +central-1|1 central-2|1 site-a-1|1 site-a-2|1 site-b-1|1 site-b-2|0 +``` + +Row restored to `4056` and a later dispatch sealed 6/6 (`080EBB0B…`, `Status=2`, every node +`Status=1`). One operational detail worth carrying: **the skip persists for up to a full +`ContactRefreshSeconds` after the row is repaired** — a deploy issued 5 s after the restore +(`DF1C0B74…`) was still skipped and still TimedOut, because the dispatch set is only rebuilt on +the refresh tick. + +**Check 8 — denials, and proof the planes were actually used.** + +``` +$ for c in central-1 central-2 site-a-1 site-a-2 site-b-1 site-b-2; do … done +central-1 telemetry=0 configserve=0 localdbsync=0 mesh=0 +central-2 telemetry=0 configserve=0 localdbsync=0 mesh=0 +site-a-1 telemetry=0 configserve=0 localdbsync=0 mesh=0 +site-a-2 telemetry=0 configserve=0 localdbsync=0 mesh=0 +site-b-1 telemetry=0 configserve=0 localdbsync=0 mesh=0 +site-b-2 telemetry=0 configserve=0 localdbsync=0 mesh=0 + +# config-serve carried real, authenticated traffic: +site-a-1 [12:20:59 INF] Fetched + verified artifact for 8d486c6b… from http://central-1:4055 (28313 bytes). +# telemetry likewise, end to end into the UI: +$ for c in …; do grep -c "Telemetry stream opened"; done → 3,4,3,3,3,5 +/hosts → CLUSTER SITE-A … sa-modbus2 SITEA-modbus Modbus HEALTHY 1 host 1 tag 12:30:49 UTC 0 errors +``` + +The five telemetry `failed … reconnecting` warnings in the run are all accounted for: three at +12:15:20 (central dialing nodes that had not finished binding `:4056`) and two at 12:28:50 / +12:29:30 (the deliberate central-2 stop in finding 5). None is an auth failure. + +### Phase 1A finding 5 — the PreStart race loses on real boots, and the memoize is what saves it + +The hardening commit's premise is not hypothetical. On the run-2 boot, **central-1 lost the +race on both dependencies**: + +``` +$ docker logs otopcua-dev-central-1-1 | grep -A3 "Failed to resolve" +[12:15:19 ERR] Failed to resolve IMeshCommandDialer for the mesh command boundary; continuing + without it. The guards below report what that costs +Akka.Hosting.MissingActorRegistryEntryException: No actor registered for key + ZB.MOM.WW.OtOpcUa.ControlPlane.AuditWriterActorKey + at ZB.MOM.WW.OtOpcUa.AdminUI.Audit.ActorAuditWriter..ctor(ActorRegistry registry) +[12:15:19 ERR] Failed to resolve IMeshPskProvider for the mesh command boundary; …same cause… +``` + +It is not rare: across the three admin boots observed in run 2 (central-1 and central-2 at +12:15, central-2 again at 12:29 after the stop/start), **two of three lost it**. Had the fix +stopped at PreStart-only resolution, those nodes would have held a permanent silent `null` — +no dialer, and a never-published known-cluster gate — and the failure would only have surfaced +the first time the coordinator singleton moved onto such a node. + +**The memoize heals it, verified on the node that actually lost the race.** Run 2's dispatches +were all fanned by central-2 (the singleton host), so central-1's dialer was never exercised +by them. Central-2 was therefore stopped to move the singleton, and a dispatch driven while +central-1 alone held it: + +``` +$ docker compose stop central-2 # 12:28:39 +$ curl … /health/ready → self akka.tcp://otopcua@central-1:4053, members 1 # 12:28:57 +$ curl -X POST /api/deployments -d '{"createdBy":"1a-gate-central1-heal"}' # 12:29:01 +$ docker logs otopcua-dev-central-1-1 | grep -E "accepted by|no command dialer is wired" +[12:29:02 DBG] Mesh: DispatchDeployment (correlationId=ad97d386…) accepted by site-b-1:4053 over gRPC +[12:29:02 DBG] … accepted by site-a-2 … site-b-2 … site-a-1 … central-1 … # 5 live nodes +# no "no command dialer is wired" line, and no further "Failed to resolve" +$ docker compose start central-2 → members 2, unreachable 0, status Healthy # 12:29:51 +``` + +So the boot-time Error is **cosmetically alarming but functionally healed**, and that is now an +observed property rather than a design claim. Two follow-ups for Phase 2 rather than for 1A: +the Error text says "continuing without it", which reads as permanent and should say that the +first use retries; and a `Failed to resolve` line on a healthy node is exactly the kind of +signal a soak's PSK-drift/error grep will trip over. + +**One incidental observation, not a check.** The deployment `RevisionHash` is deterministic +(three separate no-change dispatches all produced `642b9d6f79b9…`), but restoring the alarm +predicate's two text columns did **not** restore the pre-gate hash `6178a8f302aa…` — so the +artifact covers a column the `UPDATE` bumped (`Script.RowVersion` is the obvious candidate). +The rig's configuration content is byte-identical to its pre-gate state; only the hash moved. +Not investigated further — out of this program's scope. + +### Run 1 record — the defect that blocked it Branch `feat/grpc-node-command` @ `fbf7b708`. Image rebuilt from this worktree (`/Users/dohertj2/Desktop/OtOpcUa/.worktrees/grpc-1a`) so it carries this branch; @@ -170,26 +433,18 @@ on the compose default `ClusterClient`** (coexistence is the point). Three Seril rig's default Serilog level is `Information`, so without them checks 1/3 would have had no observable signal. Flip and overrides reverted after the gate, never committed. -**Verdict: FAIL.** Neither central node boots on this branch, so nothing that originates at -central could be driven. Root cause is defect 4 below — resolving `IMeshCommandDialer` / -`IMeshPskProvider` inside the first `WithActors` starter drags in `ISecretResolver` → +**Run 1 verdict: FAIL.** Neither central node booted, so nothing that originates at central +could be driven: checks 1, 2 and 7 were **BLOCKED** outright, 3 and 4 reached only their +node-side legs, and check 8's zero-denial result was vacuous for telemetry and config-serve +(central dials both and never booted). Checks 5 and 6 **PASSED** and are the run-1 evidence +carried into the combined table above. Root cause was defect 4 — resolving `IMeshCommandDialer` +/ `IMeshPskProvider` inside the first `WithActors` starter drags in `ISecretResolver` → `IAuditWriter` → `ActorAuditWriter`, whose constructor reads `AuditWriterActorKey` out of the -`ActorRegistry` before the singleton that registers it has run. It is **mode-independent** -(reproduced on the same image with `MeshTransport__Mode=ClusterClient`), so it is a defect of +`ActorRegistry` before the singleton that registers it has run. It was **mode-independent** +(reproduced on the same image with `MeshTransport__Mode=ClusterClient`), so it was a defect of the branch, not of the gate's flip, and it would have failed a `ClusterClient` rig equally. -| # | Check | Result | -|---|---|---| -| 1 | Deploy dispatch from AdminUI reaches **all six** driver nodes over gRPC (per-node `Execute` log lines on every node; zero `SendToAll` lines on central) | **BLOCKED** — `POST /api/deployments` is unreachable: traefik answers `503` because neither central node has a live HTTP surface (`curl http://central-1:9000/health/ready` and `central-2` both return `000`, connection refused). Nothing dispatched. | -| 2 | Acks still return over ClusterClient (site mode unchanged) and the deployment **seals** — mixed-transport deploy proven end to end | **BLOCKED** — no deployment could be dispatched (check 1). The most recent `dbo.Deployment` row is still the pre-gate `03540DA5-…` (`Status=2`, Sealed, `2026-08-06 06:11:22`), i.e. nothing was attempted during this run. | -| 3 | `RestartDriver` + `ReconnectDriver` from AdminUI land on the target site node (EventStream delivery observed via driver restart in logs) | **PARTIAL PASS (node-side leg only)** — both commands were delivered to site-a-1 over real Kestrel h2c with the correct PSK and both acted: `DriverHost site-a-1:4053: restarting driver SITEA-modbus by request of 1a-gate` followed by a fresh `spawned Modbus driver SITEA-modbus`, and `reconnecting driver … ForceReconnect requested by admin; re-entering Reconnecting`. The **AdminUI half was not driven** — see the two carry-forwards below (central down; and the UI control is unreachable on this branch anyway). | -| 4 | `AlarmCommand` (ack a scripted alarm from AdminUI) lands and the condition acknowledges | **PARTIAL PASS (envelope leg only)** — the alarm arm decoded and was delivered node-side (`Mesh: accepted AlarmCommand (correlationId=39ba4511…) over gRPC`), which is the arm with no CLR correlation member and therefore the one worth wiring live. **The acknowledge itself was NOT observed**: the rig's single `dbo.ScriptedAlarm` row (`gate481-alarm`, equipment `EQ-6e09bcc2d907`) is not owned by SITE-A, and its owning node is a central one, which is down. | -| 5 | Unauthenticated `grpcurl` to a site node `:4056` `mesh_transport.v1.NodeCommandService/Execute` ⇒ `PermissionDenied`; correct dev PSK ⇒ accepted | **PASS** — 3/3 probes graded as designed: no bearer ⇒ `PermissionDenied`, correct `dev-mesh-psk-SITE-A` + empty envelope ⇒ `InvalidArgument` (**auth passed**, the decode is what failed), and a full `RestartDriver` envelope with the same key ⇒ `{}` + observed driver restart. | -| 6 | **Wrong-key isolation:** site-b's PSK presented to site-a ⇒ `PermissionDenied` (per-cluster scoping is real, not decorative) | **PASS** — both directions rejected: `dev-mesh-psk-SITE-B` → site-a-1 ⇒ `PermissionDenied`, `dev-mesh-psk-SITE-A` → site-b-1 ⇒ `PermissionDenied`, while `dev-mesh-psk-SITE-B` → site-b-1 reaches the decoder (`InvalidArgument`) — so the rejection is the key, not the reachability. | -| 7 | A node with NULL `GrpcPort` is skipped with the warn-once, fan continues to the others | **BLOCKED** — the skip is computed by `CentralCommunicationActor.AdoptGrpcTargets` on a contact refresh; no central node is running to refresh. The rig's `ClusterNode` rows were left untouched (all six still `GrpcPort=4056`) rather than mutated for a check that could not be observed. | -| 8 | Telemetry + config-serve + LocalDb sync unaffected: 0 auth failures on their interceptors across the run | **PARTIAL PASS** — 0 `Rejected a telemetry stream call` / `Rejected a config-serve call` / `Rejected a LocalDb sync call` lines across all six nodes, and the only `Rejected a mesh command call` lines are the 3 deliberate negative probes (2 on site-a-1, 1 on site-b-1). **But telemetry and config-serve carried no traffic at all** in this run — central dials both and never booted (site-a-1 logs 0 telemetry lines) — so "unaffected" is established for LocalDb sync only. | - -### Phase 1A evidence +### Run 1 evidence **Provenance** — flip in effect on central only, sites untouched: @@ -322,7 +577,7 @@ The `TwoNodeClusterHarness` I-1 null-provider Error (a known test-harness artifa appear in any rig log — `grep -c "IMeshPskProvider"` = 0 on all six nodes, as expected for production wiring where every admin node registers the provider. -### Phase 1A defect +### Phase 1A defect — **CLOSED by `c0475f2d` + `e54883a8`, re-gated in run 2** **Defect 4 — an admin node cannot boot: eager DI resolution inside the first Akka starter.** `WithOtOpcUaControlPlaneSingletons`'s `WithActors` block resolves `IMeshCommandDialer` and @@ -351,20 +606,46 @@ Three things make this worth a pinned regression rather than a one-line fix note The shape of the fix is the one this file already uses two lines above for the coordinator proxy — hand the actor a lazy accessor (`Func` / `Func`, -or resolve on first use) so nothing is constructed while the `ActorSystem` is still building — -but the fix and its red-first pin belong to 1A, not to this record. +or resolve on first use) so nothing is constructed while the `ActorSystem` is still building. -### Rig state after the gate +**That is what shipped.** `c0475f2d` moved both dependencies behind `Func<>` invoked in +`PreStart`, pinned by `ControlPlaneStarterOrderTests` (boots a real admin host through the +production starter order and asserts `StartAsync`; verified RED against the eager resolve in +3/3 modes). `e54883a8` then made the resolution memoize-on-first-use rather than PreStart-only, +because `PreStart` runs concurrently with the remaining starters and can still lose the race — +which **run 2 observed happening on 2 of 3 real admin boots**, so that hardening was not +theoretical. Both are re-gated above; see finding 5 for the live self-heal proof. -The compose flip was reverted (`git checkout docker-dev/docker-compose.yml`) and the six nodes -recreated on the committed all-`ClusterClient` compose. Because defect 4 is mode-independent, -the branch image cannot produce a healthy central pair on *any* mode, so the rig was returned -to its pre-gate image rather than left with a dead central pair: the 1A build was preserved as -`otopcua-host:grpc-1a` (`69360a173f1f`) and the pre-gate build (`8bd81d8aea9b`, 04:52) re-tagged -back to `otopcua-host:dev`. **Anyone re-running this gate must rebuild — the running rig is -NOT on the branch under test.** Verified after restore: six nodes `running restarts=0`, zero -`Hosting failed to start` on all six, `/health/ready` 200 on every node, MAIN mesh -`memberCount=2 unreachableCount=0`, AdminUI 200 on `http://localhost:9200`. +### Rig state after run 1 (superseded by the run-2 restoration below) + +The compose flip was reverted and the six nodes recreated on the committed all-`ClusterClient` +compose. Because defect 4 was mode-independent, the branch image could not produce a healthy +central pair on *any* mode, so the rig was returned to its pre-gate image rather than left with +a dead central pair: the run-1 build was preserved as `otopcua-host:grpc-1a` (`69360a173f1f`) +and the pre-gate build (`8bd81d8aea9b`) re-tagged back to `otopcua-host:dev`. + +### Rig state after run 2 — **restored, six healthy, on the branch image** + +The flip was reverted (`git checkout docker-dev/docker-compose.yml`) and the six nodes +recreated on the committed all-`ClusterClient` compose. No image substitution was needed this +time: `otopcua-host:dev` is the run-2 build of the committed source (`1d8452a29018`), and it +boots a healthy central pair, so the rig runs the code under test. The run-1 workaround tag +`otopcua-host:grpc-1a` was deleted to stop anyone reusing a stale image. + +``` +central-1 running restarts=0 mode=ClusterClient image=1d8452a29018 hostingFailed=0 +central-2 running restarts=0 mode=ClusterClient image=1d8452a29018 hostingFailed=0 +site-a-1 running restarts=0 mode=ClusterClient image=1d8452a29018 hostingFailed=0 +site-a-2 running restarts=0 mode=ClusterClient image=1d8452a29018 hostingFailed=0 +site-b-1 running restarts=0 mode=ClusterClient image=1d8452a29018 hostingFailed=0 +site-b-2 running restarts=0 mode=ClusterClient image=1d8452a29018 hostingFailed=0 +$ curl … /health/ready → status Healthy, members 2, unreachable 0 +$ grep -c "Mesh ClusterClient for cluster" → 3 on each central node (MAIN + SITE-A + SITE-B) +``` + +The three ClusterClients reappearing on the rollback is an incidental but useful result: the +dark switch still rolls back cleanly on the fixed branch, which is what Phase 2 will depend on +if its cutover has to be reversed. --- @@ -425,12 +706,13 @@ checks 6–7. Anything a phase gate could not drive gets named here with why + what proves it instead, and is re-checked in the next gate (the ScadaBridge proportionate-gate rule). -| Item | Deferred from | Why it could not be driven | Proven instead by | Re-check in | +| Item | Deferred from | Why it could not be driven | Proven instead by | Status | |---|---|---|---|---| -| Checks 1, 2, 7 in full — deploy dispatch over gRPC, mixed-transport seal, NULL-`GrpcPort` skip | 1A | Both central nodes fail to boot (defect 4); no AdminUI, no deploy API, no contact refresh | Nothing live. Unit coverage only: the fan Theory, the gate-⊋-fan fact, `A_slow_dial_does_not_park_the_actors_mailbox` | **1A gate re-run after defect 4 is fixed** — these are the checks 1A exists to prove and must not be carried past it | -| Check 3 central half — `RestartDriver`/`ReconnectDriver` **originating at the AdminUI** | 1A | (a) central down; (b) independently, `DriverStatusPanel.razor` — the only component holding the Reconnect/Restart buttons — is referenced by **no page or component in the AdminUI**, so the control is unreachable in the running UI on this branch and on `master` | The node-side leg driven live with real envelopes (check 3 evidence); central's encode covered by the T0.2 mapper goldens | 1A re-run; the orphaned panel is pre-existing and out of this program's scope — file separately if it is to be fixed | -| Check 4 acknowledge half — the condition actually acknowledging | 1A | The rig's one `ScriptedAlarm` (`gate481-alarm`) is not owned by a SITE cluster, and its owning central node is down | The alarm arm decoding + delivering node-side (check 4 evidence) | 1A re-run, seeding a SITE-A-owned scripted alarm first | -| Check 8 for telemetry + config-serve | 1A | Central dials both planes and never booted, so neither carried a single call — 0 denials is vacuous for them | LocalDb sync (pair-local) did carry traffic with 0 denials | 1A re-run | +| Checks 1, 2, 7 in full — deploy dispatch over gRPC, mixed-transport seal, NULL-`GrpcPort` skip | 1A run 1 | Both central nodes failed to boot (defect 4) | — | **CLOSED in 1A run 2** — all three PASS live | +| Check 4 acknowledge half — the condition actually acknowledging | 1A run 1 | The rig's one `ScriptedAlarm` sits Normal and its owning central node was down | — | **CLOSED in 1A run 2** — acked from the real `/alerts` page, `ACKNOWLEDGED` row observed | +| Check 8 for telemetry + config-serve | 1A run 1 | Central dials both planes and never booted, so neither carried a call — 0 denials was vacuous | — | **CLOSED in 1A run 2** — both planes carried authenticated traffic with 0 denials | +| Check 3 central half — `RestartDriver`/`ReconnectDriver` **originating at the AdminUI** | 1A runs 1 + 2 | **Structurally undrivable, not a rig problem:** `` + `PreStart`, pinned red-first by `ControlPlaneStarterOrderTests`) + `e54883a8` (memoize-on-first-use). Re-gated in run 2: 6/6 nodes boot, 0 `Hosting failed to start` | +| 5 | 1A gate run 2 | **The `PreStart` resolve loses its race on real boots** — 2 of 3 observed admin boots logged `Failed to resolve IMeshCommandDialer` / `IMeshPskProvider` with the same `AuditWriterActorKey` cause | `PreStart` runs on a dispatcher thread concurrently with the builder's remaining starters, so it can still precede `AuditWriterActorKey`'s registration | **Not a defect — it is the case `e54883a8` was written for, and the memoize demonstrably heals it.** Proven live by moving the coordinator singleton onto the node that lost the race (stop central-2) and watching it fan a dispatch to all five live nodes with no dialer error. Residual is cosmetic: the Error text and its effect on soak greps → Phase 2 | diff --git a/docs/plans/2026-08-06-clusterclient-to-grpc-tracking.md b/docs/plans/2026-08-06-clusterclient-to-grpc-tracking.md index 89709602..de08ff80 100644 --- a/docs/plans/2026-08-06-clusterclient-to-grpc-tracking.md +++ b/docs/plans/2026-08-06-clusterclient-to-grpc-tracking.md @@ -23,7 +23,7 @@ posture). | Phase | Scope | Branch | Status | PR / merge SHA | Live gate | |---|---|---|---|---|---| | 0 | Proto + mappers + goldens; both auth interceptors; `IMeshPskProvider`; options/validators (+`Grpc` value, SplitTopology relaxation); rig dark config; stale-comment fixes; FleetStatus issue filed | `feat/grpc-phase0-contract-auth` | **MERGED to `master`** @ `10b6d209` (2026-08-06) — all 6 tasks complete @ `23555ed0`; every task spec ✅ + code ✅ (Batch B on Opus with a closed fix loop); issues #542 + #543 filed | `10b6d209` | **PASS 4/4** | -| 1A | Central→node command plane (`NodeCommandService` on the Telemetry listener; `MeshGrpcCommandDialer`; third mode branch in `CentralCommunicationActor`) — **worktree A** | `feat/grpc-node-command` | **1A.3 gate RAN 2026-08-06 @ `fbf7b708` and FAILED — defect 4 blocks the phase.** Both central nodes fail to boot (mode-independent), so checks 1/2/7 could not be driven at all and 3/4 only node-side; 5/6 PASS. 1A.1 ✅ `64774652` + 1A.2 ✅ `6a862011` (+fixes `48dc2120`/`86633f14`) all reviewed spec ✅ + code ✅. **Next: fix defect 4 with a red-first pin, then RE-RUN the 1A gate** | — | **FAIL** (5 of 8 not provable; 2 PASS, 2 partial, 3 blocked) | +| 1A | Central→node command plane (`NodeCommandService` on the Telemetry listener; `MeshGrpcCommandDialer`; third mode branch in `CentralCommunicationActor`) — **worktree A** | `feat/grpc-node-command` | **COMPLETE @ `e54883a8` — gate re-run PASSED 2026-08-06.** Run 1 @ `fbf7b708` FAILED on defect 4 (no admin node could boot, mode-independent); fixed by `c0475f2d` + `e54883a8` and re-gated. 1A.1 ✅ `64774652` + 1A.2 ✅ `6a862011` (+fixes `48dc2120`/`86633f14`) all reviewed spec ✅ + code ✅ | — | **PASS 8/8** (check 3's AdminUI half closed by composition — the control is structurally unreachable, see carry-forwards) | | 1B | Node→central ack plane (`CentralCommandService` on the ConfigServe listener; `GrpcApplyAckSender`; third branch in `NodeCommunicationActor`) — **worktree B**, rebases on 1A | `feat/grpc-central-ack` | **1B.1 ✅ `14dd07d2` + 1B.2 ✅ `a5a789d4` (+fixes `e54711a2`/`4d607575`)** — all reviews ✅ (1B.2 on Opus: 1 Critical found+fixed, see defect log); awaiting 1A merge → rebase → 1B.3 gate | — | PENDING | | 2 | Baseline reproduction on pre-flip build, then full-rig cutover (`OTOPCUA_MESH_MODE` default → `Grpc`) + soak + kill drills | (rig commit) | NOT STARTED | — | PENDING | | 3 | Delete ClusterClient (code/HOCON/config/tests); SplitTopology requires `Grpc`; grep-gates; docs sweep incl. scadaproj index | `feat/grpc-phase3-deletion` | NOT STARTED | — | — | @@ -129,7 +129,8 @@ Defects found during execution (live-gate or review), with root cause + pinned t |---|---|---|---|---| | 1 | 1B.2 | **Failover silently defeated on a hanging central** (CRITICAL, caught by Opus code review): the ack deadline was one absolute `DateTime` shared across the endpoint walk — a hung (not refused) central-1 burned the whole budget and central-2 was dialed with an expired deadline | deadline hoisted outside the `foreach`; a kill-style drill can't see it (refused = fast-fail), only a hang discriminates | `e54711a2`; pinned by `Each_endpoint_gets_a_FRESH_deadline` (mutation-verified); 1B.3 gains a `docker pause` hang drill | | 2 | 1A.2 | **Dark-switch rollback deactivated the known-cluster gate** (Opus review, I-2): gate published only under `Mode=Grpc` while the interceptor + service are port-mapped in every mode — a rollback to ClusterClient/Dps would silently restore the timing oracle + pre-auth flood | gate activation coupled to transport mode instead of the endpoint it protects | `48dc2120` — gate follows the endpoint; pinned by the every-mode Theory + Dps guard pin `86633f14` | -| 4 | 1A.3 (live gate) | **An admin node cannot boot** (BLOCKS 1A): both central nodes die at startup with `MissingActorRegistryEntryException: ZB.MOM.WW.OtOpcUa.ControlPlane.AuditWriterActorKey`, taking `AkkaHostedService` and the whole host with them. Mode-INDEPENDENT — reproduced on the same image with `MeshTransport__Mode=ClusterClient`, so it is not the gate's flip. Driver-only nodes are unaffected (they register neither `IMeshPskProvider` nor `ActorAuditWriter`) | `WithOtOpcUaControlPlaneSingletons`'s first `WithActors` starter eagerly resolves `IMeshCommandDialer` + `IMeshPskProvider` to pass into `CentralCommunicationActor.Props`. That graph is dialer → `IMeshPskProvider` → `ISecretResolver` → (`AddZbSecrets`) → `IAuditWriter` → `ActorAuditWriter`, whose ctor calls `registry.Get()` — and `WithSingleton` is registered ~60 lines later in the same builder. Same class as scadaproj#1 and mesh Phase 6's `IClusterRoleInfo` crash: only visible with the DI graph built in a real host, which is why the container-activation test (no `AddZbSecrets` in its collection) and `TwoNodeClusterHarness` (no provider at all) both miss it. Aggravating: the failed process does not exit, so `docker ps` reports `Up restarts=0` while the node serves no HTTP | **OPEN.** Fix + red-first pin belong to 1A (the gate made no product change). Suggested shape: hand the actor lazy accessors, mirroring the `Func<>` the same block already uses for the coordinator proxy. Pin must build the graph in starter order (a Host.IntegrationTests boot of an admin-role host), not merely activate the dialer | +| 5 | 1A.3 (live gate, run 2) | **The `PreStart` resolve loses its race on real boots** — 2 of 3 observed admin boots logged `Failed to resolve IMeshCommandDialer` / `IMeshPskProvider`, same `AuditWriterActorKey` cause. NOT a defect: it is exactly the case the review-prescribed hardening was written for | `PreStart` runs on a dispatcher thread concurrently with the builder's remaining starters, so it can still precede starter #4 | **Closed by `e54883a8`, and the heal is PROVEN LIVE** — the coordinator singleton was moved onto the node that lost the race (stop central-2) and it fanned a dispatch to all five live nodes with no dialer error. Residual is cosmetic (Error text reads permanent; will trip a soak error-grep on a healthy node) → Phase 2 | +| 4 | 1A.3 (live gate, run 1) | **An admin node cannot boot** (BLOCKED 1A): both central nodes die at startup with `MissingActorRegistryEntryException: ZB.MOM.WW.OtOpcUa.ControlPlane.AuditWriterActorKey`, taking `AkkaHostedService` and the whole host with them. Mode-INDEPENDENT — reproduced on the same image with `MeshTransport__Mode=ClusterClient`, so it is not the gate's flip. Driver-only nodes are unaffected (they register neither `IMeshPskProvider` nor `ActorAuditWriter`) | `WithOtOpcUaControlPlaneSingletons`'s first `WithActors` starter eagerly resolves `IMeshCommandDialer` + `IMeshPskProvider` to pass into `CentralCommunicationActor.Props`. That graph is dialer → `IMeshPskProvider` → `ISecretResolver` → (`AddZbSecrets`) → `IAuditWriter` → `ActorAuditWriter`, whose ctor calls `registry.Get()` — and `WithSingleton` is registered ~60 lines later in the same builder. Same class as scadaproj#1 and mesh Phase 6's `IClusterRoleInfo` crash: only visible with the DI graph built in a real host, which is why the container-activation test (no `AddZbSecrets` in its collection) and `TwoNodeClusterHarness` (no provider at all) both miss it. Aggravating: the failed process does not exit, so `docker ps` reports `Up restarts=0` while the node serves no HTTP | **OPEN.** Fix + red-first pin belong to 1A (the gate made no product change). Suggested shape: hand the actor lazy accessors, mirroring the `Func<>` the same block already uses for the coordinator proxy. Pin must build the graph in starter order (a Host.IntegrationTests boot of an admin-role host), not merely activate the dialer | | 3 | 1A.2 | **Fan parked the actor mailbox** (self-caught by the implementer): dialing from the receive ran `TryGetPsk`'s cold-cache 3 s block pre-await (cold after EVERY refresh due to `InvalidateAll`), stalling deployment acks behind the fan | async methods run synchronously to the first await | `Task.Run` hop (token None — a canceled scheduling would read as a DB failure via PipeTo); pinned red-first by `A_slow_dial_does_not_park_the_actors_mailbox` | ## Carry-forward register @@ -142,9 +143,10 @@ Checks a gate could not drive, named + carried to the next gate (proportionate-g | I-4 measurement: ack burst within ~3 s after a contact-refresh tick (cold PSK cache on the inbound auth path) | 1A.2 review | bounded 3 s store read + `Lazy` stampede suppression | 1B.3 gate (observe) | | `TwoNodeClusterHarness` logs the I-1 null-provider Error at every spawn (harness registers no `IMeshPskProvider`) — EXPECTED, not a rig fault | 1A.2 re-review | production pairing is `hasAdmin`-unconditional in one block | note for 1B.3/Phase 2 readers | | Register item 3 (DI role-scoping pin: admin resolves provider+dialer, driver-only does NOT) — container-activation test exists, role-scoping pin outstanding | 1A.2 | `It_activates_from_the_container_despite_the_optional_client_factory_parameter` | Phase 2 (or 1B.3) — note defect 4 shows the missing pin is not merely role-scoping but **starter-order**: the container-activation test resolves from a collection without `AddZbSecrets` | -| 1A checks 1/2/7 (deploy fan over gRPC, mixed-transport seal, NULL-`GrpcPort` skip) and the central halves of 3/4 | 1A.3 gate | Nothing live — unit coverage only | **1A gate re-run after defect 4**; must not be carried past 1A | -| Check 8 is vacuous for telemetry + config-serve — central never booted, so neither plane carried a call | 1A.3 gate | LocalDb sync did carry traffic, 0 denials | 1A gate re-run | -| **`DriverStatusPanel.razor` is referenced by no page or component** — the Reconnect/Restart buttons are unreachable in the running AdminUI (pre-existing on `master`, not caused by this program) | 1A.3 gate | Node-side leg driven with real envelopes | Out of program scope — file separately if the control is meant to exist | +| 1A checks 1/2/7 (deploy fan over gRPC, mixed-transport seal, NULL-`GrpcPort` skip) and the central half of 4 | 1A.3 gate run 1 | — | **CLOSED in 1A.3 run 2** — all driven live and PASS | +| Check 8 is vacuous for telemetry + config-serve — central never booted, so neither plane carried a call | 1A.3 gate run 1 | — | **CLOSED in 1A.3 run 2** — both planes carried authenticated traffic (28 KB artifact fetches from `:4055`, telemetry streams open on all six) with 0 denials | +| **`DriverStatusPanel.razor` is referenced by no page or component** — the Reconnect/Restart buttons are unreachable in the running AdminUI (pre-existing on `master`, not caused by this program; re-verified in run 2: 0 ` Date: Thu, 6 Aug 2026 08:39:45 -0400 Subject: [PATCH 12/12] =?UTF-8?q?docs(plans):=20Phase=201A=20close-out=20?= =?UTF-8?q?=E2=80=94=20carry-forwards=20from=20the=20gate,=20#546=20filed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plans/2026-08-06-clusterclient-to-grpc-tracking.md | 6 +++++- docs/plans/2026-08-06-clusterclient-to-grpc.md.tasks.json | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/plans/2026-08-06-clusterclient-to-grpc-tracking.md b/docs/plans/2026-08-06-clusterclient-to-grpc-tracking.md index de08ff80..8bf888fb 100644 --- a/docs/plans/2026-08-06-clusterclient-to-grpc-tracking.md +++ b/docs/plans/2026-08-06-clusterclient-to-grpc-tracking.md @@ -142,7 +142,10 @@ Checks a gate could not drive, named + carried to the next gate (proportionate-g | Hang-style central failover drill (`docker pause`, not `docker stop` — only a hang discriminates the per-endpoint-deadline fix) | 1B.2 review | unit fact `Each_endpoint_gets_a_FRESH_deadline` | 1B.3 gate (MUST include) | | I-4 measurement: ack burst within ~3 s after a contact-refresh tick (cold PSK cache on the inbound auth path) | 1A.2 review | bounded 3 s store read + `Lazy` stampede suppression | 1B.3 gate (observe) | | `TwoNodeClusterHarness` logs the I-1 null-provider Error at every spawn (harness registers no `IMeshPskProvider`) — EXPECTED, not a rig fault | 1A.2 re-review | production pairing is `hasAdmin`-unconditional in one block | note for 1B.3/Phase 2 readers | -| Register item 3 (DI role-scoping pin: admin resolves provider+dialer, driver-only does NOT) — container-activation test exists, role-scoping pin outstanding | 1A.2 | `It_activates_from_the_container_despite_the_optional_client_factory_parameter` | Phase 2 (or 1B.3) — note defect 4 shows the missing pin is not merely role-scoping but **starter-order**: the container-activation test resolves from a collection without `AddZbSecrets` | +| Register item 3 (DI role-scoping pin: admin resolves provider+dialer, driver-only does NOT) — container-activation test exists, role-scoping pin outstanding | 1A.2 | `It_activates_from_the_container_despite_the_optional_client_factory_parameter`; starter-order half now covered by `ControlPlaneStarterOrderTests` | Phase 2 (or 1B.3) — note defect 4 shows the missing pin is not merely role-scoping but **starter-order**: the container-activation test resolves from a collection without `AddZbSecrets` | +| `Resolve`-failure Error wording says "continuing without it" (reads permanent — the memoize heals it at first use) and the line fires on real boots (2/3 in the gate) — reword or whitelist it before the Phase 2 soak error-grep, else a healthy central scores as failing the soak | 1A.3 re-run | heal proven live: singleton moved onto the race-losing node, fan clean, no dialer-missing error | Phase 2 (soak grep design) | +| A repaired `GrpcPort` takes effect only at the next contact refresh (≤60 s): a deploy 5 s post-restore was still skipped and honestly TimedOut — by design, operator-visible; document if it surprises in Phase 2 | 1A.3 re-run | check 7 evidence | Phase 2 (observe) | +| `RevisionHash` moved on byte-identical config after a direct SQL `UPDATE` bumped `Script.RowVersion` (hash covers a column content-identity does not); determinism itself holds (3 no-change dispatches → identical hash) | 1A.3 re-run | incidental, out of program scope | note only | | 1A checks 1/2/7 (deploy fan over gRPC, mixed-transport seal, NULL-`GrpcPort` skip) and the central half of 4 | 1A.3 gate run 1 | — | **CLOSED in 1A.3 run 2** — all driven live and PASS | | Check 8 is vacuous for telemetry + config-serve — central never booted, so neither plane carried a call | 1A.3 gate run 1 | — | **CLOSED in 1A.3 run 2** — both planes carried authenticated traffic (28 KB artifact fetches from `:4055`, telemetry streams open on all six) with 0 denials | | **`DriverStatusPanel.razor` is referenced by no page or component** — the Reconnect/Restart buttons are unreachable in the running AdminUI (pre-existing on `master`, not caused by this program; re-verified in run 2: 0 ` run 2 PASS 8/8; issues #544 #545 #546 filed" }, { "id": 9,