using System.Diagnostics.CodeAnalysis; using Grpc.Core; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1; using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication; /// /// mesh gRPC command plane, Phase 1A — central's outbound command dial. One node, one call, no retry. /// /// /// The facts that matter are the credential ones: the bearer must be the TARGET node's cluster's /// key (not central's, and not some other cluster's), and a cluster with no resolvable key must /// never be dialled at all. Everything else is plumbing. /// public sealed class MeshGrpcCommandDialerTests { private const string Endpoint = "http://site-a-1:4056"; private static CommandEnvelope Envelope() => new() { CorrelationId = Guid.NewGuid().ToString("N"), RestartDriver = new RestartDriverCommand { ClusterId = "SITE-A", DriverInstanceId = "drv-1", ActorByUserName = "operator", CorrelationId = Guid.NewGuid().ToString("N"), }, }; private static MeshGrpcCommandDialer CreateDialer( IMeshPskProvider psk, RecordingClient client, int dispatchTimeoutSeconds = 10) => new( psk, Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = dispatchTimeoutSeconds }), NullLogger.Instance, _ => client); [Fact] public async Task Dial_presents_the_target_clusters_key_and_names_that_cluster() { // THE assertion. Central holds N keys; sending the wrong one turns a legitimate command into // a PermissionDenied, and sending a fleet-wide one would be the blast radius this design // exists to avoid. var client = new RecordingClient(); var dialer = CreateDialer( new FakePskProvider(("SITE-A", "psk-site-a"), ("SITE-B", "psk-site-b")), client); await dialer.DispatchAsync(Endpoint, Envelope(), "SITE-B", CancellationToken.None); client.LastHeaders.ShouldNotBeNull(); client.LastHeaders!.GetValue("authorization").ShouldBe("Bearer psk-site-b"); client.LastHeaders.GetValue("x-otopcua-cluster").ShouldBe("SITE-B"); } [Fact] public async Task A_cluster_with_no_key_is_refused_locally_and_never_dialled() { // Fail-closed: not "dial and let the node reject it". An unauthenticated command attempt // against a node's control surface should not exist even as a rejected one. var client = new RecordingClient(); var dialer = CreateDialer(new FakePskProvider(("SITE-A", "psk-site-a")), client); var ex = await Should.ThrowAsync( () => dialer.DispatchAsync(Endpoint, Envelope(), "SITE-Z", CancellationToken.None)); ex.StatusCode.ShouldBe(StatusCode.Unauthenticated); client.Calls.ShouldBe(0); } [Fact] public async Task The_envelope_reaches_the_node_unchanged() { var client = new RecordingClient(); var dialer = CreateDialer(new FakePskProvider(("SITE-A", "psk-site-a")), client); var envelope = Envelope(); await dialer.DispatchAsync(Endpoint, envelope, "SITE-A", CancellationToken.None); client.LastRequest.ShouldBe(envelope); } [Fact] public async Task A_node_rejection_surfaces_as_its_own_RpcException() { // The actor grades per-node outcomes off the status code, so it must arrive intact rather // than wrapped in an AggregateException or swallowed here. var client = new RecordingClient { Fault = new RpcException(new Status(StatusCode.PermissionDenied, "nope")), }; var dialer = CreateDialer(new FakePskProvider(("SITE-A", "psk-site-a")), client); var ex = await Should.ThrowAsync( () => dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", CancellationToken.None)); ex.StatusCode.ShouldBe(StatusCode.PermissionDenied); } [Fact] public async Task The_per_call_deadline_bounds_a_node_that_never_answers() { // A dispatch dials every node in the fleet. Without this bound one unreachable node holds a // task and a socket for as long as the process lives, while the deploy path's own apply // deadline expires around it. var client = new RecordingClient { HangUntilCancelled = true }; var dialer = CreateDialer( new FakePskProvider(("SITE-A", "psk-site-a")), client, dispatchTimeoutSeconds: 1); await Should.ThrowAsync( () => dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", CancellationToken.None) .WaitAsync(TimeSpan.FromSeconds(10))); } [Fact] public async Task The_callers_token_still_cancels_a_dial_inside_the_deadline() { // Linked, not replaced: actor shutdown must cancel everything in flight even when the // per-call deadline is nowhere near expiring. var client = new RecordingClient { HangUntilCancelled = true }; var dialer = CreateDialer( new FakePskProvider(("SITE-A", "psk-site-a")), client, dispatchTimeoutSeconds: 600); using var cts = new CancellationTokenSource(); var dial = dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", cts.Token); await cts.CancelAsync(); await Should.ThrowAsync(() => dial.WaitAsync(TimeSpan.FromSeconds(10))); } [Fact] public async Task A_non_positive_dispatch_timeout_is_clamped_rather_than_honoured() { // Zero would mean an UNBOUNDED call — the exact failure the deadline exists to prevent — and // MeshTransportOptionsValidator's rule for this key is scoped to driver-role nodes, so a // pure-admin central reaches this code with the value unvalidated. Clamp loudly. var client = new RecordingClient { HangUntilCancelled = true }; var logger = new RecordingLogger(); var dialer = new MeshGrpcCommandDialer( new FakePskProvider(("SITE-A", "psk-site-a")), Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 0 }), logger, _ => client); logger.Warnings.ShouldContain(w => w.Contains("DispatchTimeoutSeconds")); // And the clamp is real, not just logged: the dial must still expire on its own. await Should.ThrowAsync( () => dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", CancellationToken.None) .WaitAsync(TimeSpan.FromSeconds(20))); } [Fact] public async Task Dialing_after_disposal_throws_rather_than_repopulating_the_channel_cache() { var client = new RecordingClient(); var dialer = CreateDialer(new FakePskProvider(("SITE-A", "psk-site-a")), client); dialer.Dispose(); await Should.ThrowAsync( () => dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", CancellationToken.None)); } [Fact] public async Task The_real_channel_factory_dials_and_is_disposable() { // The fake client seam covers every other fact here, which means nothing else ever exercises // GrpcChannel.ForAddress or the cache it fills. Dial an endpoint nothing is listening on: // the failure must be an ordinary RpcException the actor can log, not a construction throw, // and Dispose must then clean up the channel the dial created. var dialer = new MeshGrpcCommandDialer( new FakePskProvider(("SITE-A", "psk-site-a")), Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 5 }), NullLogger.Instance); await Should.ThrowAsync( () => dialer.DispatchAsync( "http://127.0.0.1:1", Envelope(), "SITE-A", CancellationToken.None)); Should.NotThrow(dialer.Dispose); } [Fact] public async Task Two_dials_to_one_endpoint_share_a_single_channel() { // The cache is the whole defence against a socket per call: a deploy dispatch fans across // every node in the fleet, and it repeats on every redeploy. A channel per dial exhausts // ports on central long before anything else notices. var dialer = new MeshGrpcCommandDialer( new FakePskProvider(("SITE-A", "psk-site-a")), // 1 s, not 5: on a host that blackholes loopback instead of refusing, each dial costs // the full deadline — keep the worst case bounded so this never reads as a hang. Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 1 }), NullLogger.Instance); // Nothing is listening, so both dials fail — irrelevant here: the channel is constructed // before the call is placed, which is exactly what is being counted. for (var i = 0; i < 2; i++) { await Should.ThrowAsync( () => dialer.DispatchAsync( "http://127.0.0.1:1", Envelope(), "SITE-A", CancellationToken.None)); } dialer.CachedChannelCount.ShouldBe(1); // A second endpoint is a second channel — the cache keys on the endpoint, it does not // collapse every node onto one connection. await Should.ThrowAsync( () => dialer.DispatchAsync( "http://127.0.0.1:2", Envelope(), "SITE-A", CancellationToken.None)); dialer.CachedChannelCount.ShouldBe(2); Should.NotThrow(dialer.Dispose); } [Fact] public void It_activates_from_the_container_despite_the_optional_client_factory_parameter() { // The seam is an optional constructor parameter, which MS.DI honours only because it checks // HasDefaultValue. Get that wrong — by making it non-optional, or by adding a second public // constructor — and the failure is a startup crash on the admin node, invisible to every // other test here because they all construct the dialer by hand. var services = new ServiceCollection(); services.AddLogging(); services.AddSingleton(new FakePskProvider(("SITE-A", "psk-site-a"))); services.AddSingleton>( Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 10 })); services.AddSingleton(); using var provider = services.BuildServiceProvider(); provider.GetRequiredService().ShouldBeOfType(); } /// /// A that records what it was /// handed and answers however the test asks. The generated client's protected parameterless /// constructor exists precisely for this. /// private sealed class RecordingClient : NodeCommandService.NodeCommandServiceClient { /// Number of calls that reached the client. public int Calls { get; private set; } /// Headers of the most recent call. public Metadata? LastHeaders { get; private set; } /// Request of the most recent call. public CommandEnvelope? LastRequest { get; private set; } /// When set, the call faults with this instead of answering. public RpcException? Fault { get; set; } /// When true, the call never completes until its token is cancelled. public bool HangUntilCancelled { get; set; } public override AsyncUnaryCall ExecuteAsync( CommandEnvelope request, Metadata? headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default) { Calls++; LastHeaders = headers; LastRequest = request; var response = Fault is not null ? Task.FromException(Fault) : HangUntilCancelled ? HangAsync(cancellationToken) : Task.FromResult(new ExecuteAck()); return new AsyncUnaryCall( response, Task.FromResult(new Metadata()), () => Status.DefaultSuccess, () => [], () => { }); } private static async Task HangAsync(CancellationToken ct) { await Task.Delay(Timeout.InfiniteTimeSpan, ct).ConfigureAwait(false); return new ExecuteAck(); } } /// Captures Warning-level messages so a construction-time check is assertable. private sealed class RecordingLogger : ILogger { /// Every warning logged, formatted. public List Warnings { get; } = []; public IDisposable? BeginScope(TState state) where TState : notnull => null; public bool IsEnabled(LogLevel logLevel) => true; public void Log( LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) { if (logLevel == LogLevel.Warning) Warnings.Add(formatter(state, exception)); } } /// An over a fixed key set; unknown clusters fail closed. private sealed class FakePskProvider(params (string Cluster, string Psk)[] keys) : IMeshPskProvider { private readonly Dictionary _keys = keys.ToDictionary(k => k.Cluster, k => k.Psk, StringComparer.OrdinalIgnoreCase); public bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk) => _keys.TryGetValue(clusterId, out psk); public bool Matches(string clusterId, string presented) => _keys.TryGetValue(clusterId, out var expected) && string.Equals(expected, presented, StringComparison.Ordinal); public void SetKnownClusters(IReadOnlyCollection clusterIds) { } public void Invalidate(string clusterId) { } public void InvalidateAll() { } } }