lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs

using System.Diagnostics.CodeAnalysis;
using Grpc.Core;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;

namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication;

/// <summary>
///     mesh gRPC command plane, Phase 1A — central's outbound command dial. One node, one call, no retry.
/// </summary>
/// <remarks>
///     The facts that matter are the credential ones: the bearer must be the TARGET node's cluster's
///     key (not central's, and not some other cluster's), and a cluster with no resolvable key must
///     never be dialled at all. Everything else is plumbing.
/// </remarks>
public sealed class MeshGrpcCommandDialerTests
{
    private const string Endpoint = "http://site-a-1:4056";

    private static CommandEnvelope Envelope() => new()
    {
        CorrelationId = Guid.NewGuid().ToString("N"),
        RestartDriver = new RestartDriverCommand
        {
            ClusterId = "SITE-A",
            DriverInstanceId = "drv-1",
            ActorByUserName = "operator",
            CorrelationId = Guid.NewGuid().ToString("N"),
        },
    };

    private static MeshGrpcCommandDialer CreateDialer(
        IMeshPskProvider psk,
        RecordingClient client,
        int dispatchTimeoutSeconds = 10)
        => new(
            psk,
            Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = dispatchTimeoutSeconds }),
            NullLogger<MeshGrpcCommandDialer>.Instance,
            _ => client);

    [Fact]
    public async Task Dial_presents_the_target_clusters_key_and_names_that_cluster()
    {
        // THE assertion. Central holds N keys; sending the wrong one turns a legitimate command into
        // a PermissionDenied, and sending a fleet-wide one would be the blast radius this design
        // exists to avoid.
        var client = new RecordingClient();
        var dialer = CreateDialer(
            new FakePskProvider(("SITE-A", "psk-site-a"), ("SITE-B", "psk-site-b")), client);

        await dialer.DispatchAsync(Endpoint, Envelope(), "SITE-B", CancellationToken.None);

        client.LastHeaders.ShouldNotBeNull();
        client.LastHeaders!.GetValue("authorization").ShouldBe("Bearer psk-site-b");
        client.LastHeaders.GetValue("x-otopcua-cluster").ShouldBe("SITE-B");
    }

    [Fact]
    public async Task A_cluster_with_no_key_is_refused_locally_and_never_dialled()
    {
        // Fail-closed: not "dial and let the node reject it". An unauthenticated command attempt
        // against a node's control surface should not exist even as a rejected one.
        var client = new RecordingClient();
        var dialer = CreateDialer(new FakePskProvider(("SITE-A", "psk-site-a")), client);

        var ex = await Should.ThrowAsync<RpcException>(
            () => dialer.DispatchAsync(Endpoint, Envelope(), "SITE-Z", CancellationToken.None));

        ex.StatusCode.ShouldBe(StatusCode.Unauthenticated);
        client.Calls.ShouldBe(0);
    }

    [Fact]
    public async Task The_envelope_reaches_the_node_unchanged()
    {
        var client = new RecordingClient();
        var dialer = CreateDialer(new FakePskProvider(("SITE-A", "psk-site-a")), client);
        var envelope = Envelope();

        await dialer.DispatchAsync(Endpoint, envelope, "SITE-A", CancellationToken.None);

        client.LastRequest.ShouldBe(envelope);
    }

    [Fact]
    public async Task A_node_rejection_surfaces_as_its_own_RpcException()
    {
        // The actor grades per-node outcomes off the status code, so it must arrive intact rather
        // than wrapped in an AggregateException or swallowed here.
        var client = new RecordingClient
        {
            Fault = new RpcException(new Status(StatusCode.PermissionDenied, "nope")),
        };
        var dialer = CreateDialer(new FakePskProvider(("SITE-A", "psk-site-a")), client);

        var ex = await Should.ThrowAsync<RpcException>(
            () => dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", CancellationToken.None));

        ex.StatusCode.ShouldBe(StatusCode.PermissionDenied);
    }

    [Fact]
    public async Task The_per_call_deadline_bounds_a_node_that_never_answers()
    {
        // A dispatch dials every node in the fleet. Without this bound one unreachable node holds a
        // task and a socket for as long as the process lives, while the deploy path's own apply
        // deadline expires around it.
        var client = new RecordingClient { HangUntilCancelled = true };
        var dialer = CreateDialer(
            new FakePskProvider(("SITE-A", "psk-site-a")), client, dispatchTimeoutSeconds: 1);

        await Should.ThrowAsync<OperationCanceledException>(
            () => dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", CancellationToken.None)
                .WaitAsync(TimeSpan.FromSeconds(10)));
    }

    [Fact]
    public async Task The_callers_token_still_cancels_a_dial_inside_the_deadline()
    {
        // Linked, not replaced: actor shutdown must cancel everything in flight even when the
        // per-call deadline is nowhere near expiring.
        var client = new RecordingClient { HangUntilCancelled = true };
        var dialer = CreateDialer(
            new FakePskProvider(("SITE-A", "psk-site-a")), client, dispatchTimeoutSeconds: 600);
        using var cts = new CancellationTokenSource();

        var dial = dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", cts.Token);
        await cts.CancelAsync();

        await Should.ThrowAsync<OperationCanceledException>(() => dial.WaitAsync(TimeSpan.FromSeconds(10)));
    }

    [Fact]
    public async Task A_non_positive_dispatch_timeout_is_clamped_rather_than_honoured()
    {
        // Zero would mean an UNBOUNDED call — the exact failure the deadline exists to prevent — and
        // MeshTransportOptionsValidator's rule for this key is scoped to driver-role nodes, so a
        // pure-admin central reaches this code with the value unvalidated. Clamp loudly.
        var client = new RecordingClient { HangUntilCancelled = true };
        var logger = new RecordingLogger();
        var dialer = new MeshGrpcCommandDialer(
            new FakePskProvider(("SITE-A", "psk-site-a")),
            Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 0 }),
            logger,
            _ => client);

        logger.Warnings.ShouldContain(w => w.Contains("DispatchTimeoutSeconds"));

        // And the clamp is real, not just logged: the dial must still expire on its own.
        await Should.ThrowAsync<OperationCanceledException>(
            () => dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", CancellationToken.None)
                .WaitAsync(TimeSpan.FromSeconds(20)));
    }

    [Fact]
    public async Task Dialing_after_disposal_throws_rather_than_repopulating_the_channel_cache()
    {
        var client = new RecordingClient();
        var dialer = CreateDialer(new FakePskProvider(("SITE-A", "psk-site-a")), client);
        dialer.Dispose();

        await Should.ThrowAsync<ObjectDisposedException>(
            () => dialer.DispatchAsync(Endpoint, Envelope(), "SITE-A", CancellationToken.None));
    }

    [Fact]
    public async Task The_real_channel_factory_dials_and_is_disposable()
    {
        // The fake client seam covers every other fact here, which means nothing else ever exercises
        // GrpcChannel.ForAddress or the cache it fills. Dial an endpoint nothing is listening on:
        // the failure must be an ordinary RpcException the actor can log, not a construction throw,
        // and Dispose must then clean up the channel the dial created.
        var dialer = new MeshGrpcCommandDialer(
            new FakePskProvider(("SITE-A", "psk-site-a")),
            Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 5 }),
            NullLogger<MeshGrpcCommandDialer>.Instance);

        await Should.ThrowAsync<RpcException>(
            () => dialer.DispatchAsync(
                "http://127.0.0.1:1", Envelope(), "SITE-A", CancellationToken.None));

        Should.NotThrow(dialer.Dispose);
    }

    [Fact]
    public async Task Two_dials_to_one_endpoint_share_a_single_channel()
    {
        // The cache is the whole defence against a socket per call: a deploy dispatch fans across
        // every node in the fleet, and it repeats on every redeploy. A channel per dial exhausts
        // ports on central long before anything else notices.
        var dialer = new MeshGrpcCommandDialer(
            new FakePskProvider(("SITE-A", "psk-site-a")),
            // 1 s, not 5: on a host that blackholes loopback instead of refusing, each dial costs
            // the full deadline — keep the worst case bounded so this never reads as a hang.
            Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 1 }),
            NullLogger<MeshGrpcCommandDialer>.Instance);

        // Nothing is listening, so both dials fail — irrelevant here: the channel is constructed
        // before the call is placed, which is exactly what is being counted.
        for (var i = 0; i < 2; i++)
        {
            await Should.ThrowAsync<RpcException>(
                () => dialer.DispatchAsync(
                    "http://127.0.0.1:1", Envelope(), "SITE-A", CancellationToken.None));
        }

        dialer.CachedChannelCount.ShouldBe(1);

        // A second endpoint is a second channel — the cache keys on the endpoint, it does not
        // collapse every node onto one connection.
        await Should.ThrowAsync<RpcException>(
            () => dialer.DispatchAsync(
                "http://127.0.0.1:2", Envelope(), "SITE-A", CancellationToken.None));

        dialer.CachedChannelCount.ShouldBe(2);
        Should.NotThrow(dialer.Dispose);
    }

    [Fact]
    public void It_activates_from_the_container_despite_the_optional_client_factory_parameter()
    {
        // The seam is an optional constructor parameter, which MS.DI honours only because it checks
        // HasDefaultValue. Get that wrong — by making it non-optional, or by adding a second public
        // constructor — and the failure is a startup crash on the admin node, invisible to every
        // other test here because they all construct the dialer by hand.
        var services = new ServiceCollection();
        services.AddLogging();
        services.AddSingleton<IMeshPskProvider>(new FakePskProvider(("SITE-A", "psk-site-a")));
        services.AddSingleton<IOptions<MeshTransportOptions>>(
            Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 10 }));
        services.AddSingleton<IMeshCommandDialer, MeshGrpcCommandDialer>();

        using var provider = services.BuildServiceProvider();

        provider.GetRequiredService<IMeshCommandDialer>().ShouldBeOfType<MeshGrpcCommandDialer>();
    }

    /// <summary>
    ///     A <see cref="NodeCommandService.NodeCommandServiceClient"/> that records what it was
    ///     handed and answers however the test asks. The generated client's protected parameterless
    ///     constructor exists precisely for this.
    /// </summary>
    private sealed class RecordingClient : NodeCommandService.NodeCommandServiceClient
    {
        /// <summary>Number of calls that reached the client.</summary>
        public int Calls { get; private set; }

        /// <summary>Headers of the most recent call.</summary>
        public Metadata? LastHeaders { get; private set; }

        /// <summary>Request of the most recent call.</summary>
        public CommandEnvelope? LastRequest { get; private set; }

        /// <summary>When set, the call faults with this instead of answering.</summary>
        public RpcException? Fault { get; set; }

        /// <summary>When true, the call never completes until its token is cancelled.</summary>
        public bool HangUntilCancelled { get; set; }

        public override AsyncUnaryCall<ExecuteAck> ExecuteAsync(
            CommandEnvelope request,
            Metadata? headers = null,
            DateTime? deadline = null,
            CancellationToken cancellationToken = default)
        {
            Calls++;
            LastHeaders = headers;
            LastRequest = request;

            var response = Fault is not null
                ? Task.FromException<ExecuteAck>(Fault)
                : HangUntilCancelled
                    ? HangAsync(cancellationToken)
                    : Task.FromResult(new ExecuteAck());

            return new AsyncUnaryCall<ExecuteAck>(
                response,
                Task.FromResult(new Metadata()),
                () => Status.DefaultSuccess,
                () => [],
                () => { });
        }

        private static async Task<ExecuteAck> HangAsync(CancellationToken ct)
        {
            await Task.Delay(Timeout.InfiniteTimeSpan, ct).ConfigureAwait(false);
            return new ExecuteAck();
        }
    }

    /// <summary>Captures Warning-level messages so a construction-time check is assertable.</summary>
    private sealed class RecordingLogger : ILogger<MeshGrpcCommandDialer>
    {
        /// <summary>Every warning logged, formatted.</summary>
        public List<string> Warnings { get; } = [];

        public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;

        public bool IsEnabled(LogLevel logLevel) => true;

        public void Log<TState>(
            LogLevel logLevel,
            EventId eventId,
            TState state,
            Exception? exception,
            Func<TState, Exception?, string> formatter)
        {
            if (logLevel == LogLevel.Warning)
                Warnings.Add(formatter(state, exception));
        }
    }

    /// <summary>An <see cref="IMeshPskProvider"/> over a fixed key set; unknown clusters fail closed.</summary>
    private sealed class FakePskProvider(params (string Cluster, string Psk)[] keys) : IMeshPskProvider
    {
        private readonly Dictionary<string, string> _keys =
            keys.ToDictionary(k => k.Cluster, k => k.Psk, StringComparer.OrdinalIgnoreCase);

        public bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk)
            => _keys.TryGetValue(clusterId, out psk);

        public bool Matches(string clusterId, string presented)
            => _keys.TryGetValue(clusterId, out var expected)
               && string.Equals(expected, presented, StringComparison.Ordinal);

        public void SetKnownClusters(IReadOnlyCollection<string> clusterIds) { }

        public void Invalidate(string clusterId) { }

        public void InvalidateAll() { }
    }
}