lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/MeshGrpcBoundaryTests.cs

using System.Diagnostics.CodeAnalysis;
using System.Net;
using Akka.Actor;
using Akka.Cluster;
using Akka.Configuration;
using Akka.Hosting;
using Grpc.Core;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Server.Kestrel.Core;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
using ZB.MOM.WW.OtOpcUa.Host.Grpc;
using ZB.MOM.WW.OtOpcUa.Runtime;
using ZB.MOM.WW.OtOpcUa.Runtime.Communication;
using AkkaCluster = Akka.Cluster.Cluster;

namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;

/// <summary>
///     Drives the mesh command plane across two genuinely separate <see cref="ActorSystem"/>s over
///     a real gRPC wire — the only test in the repo where the central↔node boundary carries a
///     message off-process.
/// </summary>
/// <remarks>
///     <para>
///         Replaces <c>MeshClusterClientBoundaryTests</c>, which did the same job for the Akka
///         ClusterClient transport that this program deleted. The reason it exists is unchanged:
///         a fake dialer tests actor <i>logic</i> and cannot fail for any transport reason — not a
///         service that was never mapped, not an interceptor that refuses the call, not a
///         serialization break. Those are the failure modes this class exercises, end to end
///         through the REAL <c>CentralCommunicationActor</c>, <c>MeshGrpcCommandDialer</c>,
///         <c>MeshCommandAuthInterceptor</c>, <c>NodeCommandGrpcService</c> and
///         <c>NodeCommunicationActor</c>.
///     </para>
///     <para>
///         <b>Three clusters, one ActorSystem name, joined ONLY by gRPC.</b> The shared name is kept
///         from the predecessor so the shape is comparable; each system seeds itself and none lists
///         another, and
///         <see cref="No_cross_mesh_Akka_association_forms_between_central_and_either_site"/>
///         asserts the separation outright. That assertion is the whole point of the program — the
///         cross-WAN Akka association is gone — so it is kept as a test rather than left to a live
///         gate: if the three systems ever gossiped into one mesh, every delivery below could be
///         plain local messaging and the class would be testing nothing.
///     </para>
///     <para>
///         <b>Per-cluster keys, not one fleet key.</b> Each site node's interceptor is configured
///         with its OWN <c>MeshTransport:GrpcPsk</c> and central's provider hands the dialer the key
///         for the cluster it is dialling. A fan that presented the wrong cluster's key would be
///         refused at the far side, so the happy path below also pins that the dialer selects by
///         <c>ClusterId</c>.
///     </para>
/// </remarks>
public sealed class MeshGrpcBoundaryTests : IAsyncLifetime
{
    private const string SharedSystemName = "otopcua";
    private const string SiteAKey = "psk-site-a";
    private const string SiteBKey = "psk-site-b";

    private static readonly TimeSpan Arrival = TimeSpan.FromSeconds(30);

    /// <summary>
    ///     How long the falsifiability control waits before concluding nothing arrived. Deliberately
    ///     shorter than <see cref="Arrival"/> but far longer than a successful delivery takes in
    ///     practice, so a pass means "the boundary refused it", not "the clock ran out first".
    /// </summary>
    private static readonly TimeSpan Silence = TimeSpan.FromSeconds(10);

    private ActorSystem _central = null!;
    private SiteNode _siteA = null!;
    private SiteNode _siteB = null!;

    /// <inheritdoc />
    public async ValueTask InitializeAsync()
    {
        _central = StartSelfSeededSystem();
        _siteA = await SiteNode.StartAsync("SITE-A", SiteAKey);
        _siteB = await SiteNode.StartAsync("SITE-B", SiteBKey);

        await Task.WhenAll(
            AwaitSingleMemberUpAsync(_central),
            AwaitSingleMemberUpAsync(_siteA.System),
            AwaitSingleMemberUpAsync(_siteB.System));
    }

    /// <inheritdoc />
    public async ValueTask DisposeAsync()
    {
        await _siteA.DisposeAsync();
        await _siteB.DisposeAsync();
        await ShutdownAsync(_central);
    }

    [Fact]
    public void No_cross_mesh_Akka_association_forms_between_central_and_either_site()
    {
        // THE assertion of the ClusterClient→gRPC program, kept as a test rather than a live-gate
        // note. Three systems sharing a name in one process are one gossip accident away from being
        // one cluster, at which point the gRPC boundary is no longer the thing under test — and,
        // more importantly, the cross-WAN Akka association this program removed would be back.
        var central = AkkaCluster.Get(_central);
        var siteA = AkkaCluster.Get(_siteA.System);
        var siteB = AkkaCluster.Get(_siteB.System);

        central.State.Members.Count.ShouldBe(1,
            "central must be its own single-member cluster; commands cross to a site over gRPC only");
        siteA.State.Members.Count.ShouldBe(1, "site-A must be its own single-member cluster");
        siteB.State.Members.Count.ShouldBe(1, "site-B must be its own single-member cluster");

        // Stated separately from the counts: a count of 1 on each side would also hold in the instant
        // before a join completed, whereas naming the addresses says no side ever learned another's.
        foreach (var (name, cluster) in new[]
                 {
                     ("central", central), ("site-A", siteA), ("site-B", siteB),
                 })
        {
            var others = new[] { central.SelfAddress, siteA.SelfAddress, siteB.SelfAddress }
                .Where(a => a != cluster.SelfAddress);
            foreach (var other in others)
            {
                cluster.State.Members.ShouldNotContain(
                    m => m.Address == other,
                    $"{name} must not have {other} as a cluster member — the cross-mesh Akka "
                    + "association is exactly what the gRPC command plane replaced");
            }
        }
    }

    [Fact]
    public async Task One_dispatch_through_central_reaches_a_node_in_every_cluster_mesh()
    {
        // The direct counterpart of the deleted ClusterClient fan-out test, at the transport level and
        // driving the REAL CentralCommunicationActor. Two genuinely separate site meshes, each with
        // its own gRPC command endpoint and its own per-cluster key; one MeshCommand must reach BOTH.
        var dbFactory = SeedTwoClusterDb(_siteA, _siteB);
        var actor = _central.ActorOf(CentralCommunicationActor.Props(
            dbFactory,
            GrpcOptions(),
            () => null,
            () => Dialer(),
            () => new FixedPskProvider()));

        var command = new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid());
        var deadline = DateTime.UtcNow + Arrival;
        while (DateTime.UtcNow < deadline
               && !(_siteA.Inbox.Task.IsCompleted && _siteB.Inbox.Task.IsCompleted))
        {
            // Re-sent rather than sent once: the actor's first contact load is a piped DB read, and a
            // command told before it lands is dropped as "no targets" by design. Re-sending weakens
            // nothing — the wrong-key control below re-sends on the same schedule and must still
            // receive nothing.
            actor.Tell(new MeshCommand("deployments", command));
            await Task.Delay(250, TestContext.Current.CancellationToken);
        }

        _siteA.Inbox.Task.IsCompleted.ShouldBeTrue(
            "site-A's mesh must receive the dispatch on its own gRPC command endpoint");
        _siteB.Inbox.Task.IsCompleted.ShouldBeTrue(
            "site-B's mesh must receive the SAME dispatch on its own endpoint, under its own "
            + "per-cluster key — a fan that stopped at the first cluster would leave site-B silently "
            + "on its old configuration while the deployment sealed green");

        (await _siteA.Inbox.Task).ShouldBeOfType<RestartDriver>().DriverInstanceId.ShouldBe("driver-1");
        (await _siteB.Inbox.Task).ShouldBeOfType<RestartDriver>().DriverInstanceId.ShouldBe("driver-1");
    }

    [Fact]
    public async Task Falsifiability_control_a_dial_carrying_the_wrong_cluster_key_delivers_nothing()
    {
        // Without this, the test above cannot distinguish "the gRPC boundary carried the command"
        // from "something else in the process delivered it". Same wire, same service, same comm
        // actor — and with a key site-A's interceptor does not accept, it must NOT arrive.
        var envelope = MeshTransportDtoMapper.ToEnvelope(
            new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid()));
        var dialer = Dialer(new FixedPskProvider("not-the-key"));

        var deadline = DateTime.UtcNow + Silence;
        while (DateTime.UtcNow < deadline && !_siteA.Inbox.Task.IsCompleted)
        {
            try
            {
                await dialer.DispatchAsync(
                    _siteA.Endpoint, envelope, "SITE-A", TestContext.Current.CancellationToken);
            }
            catch (RpcException)
            {
                // Expected: the node's MeshCommandAuthInterceptor refuses the bearer. The assertion
                // is what did NOT arrive, so the rejection itself is swallowed rather than asserted
                // on here — MeshCommandAuthInterceptorTests owns the status-code contract.
            }

            await Task.Delay(500, TestContext.Current.CancellationToken);
        }

        _siteA.Inbox.Task.IsCompleted.ShouldBeFalse(
            "delivery survived presenting the wrong per-cluster key, so the positive test above is "
            + "not measuring the authenticated gRPC boundary at all");

        // Same node, same dialer shape, right key: proves the silence above was the auth gate rather
        // than a dead endpoint or a misaddressed call.
        await Dialer().DispatchAsync(
            _siteA.Endpoint, envelope, "SITE-A", TestContext.Current.CancellationToken);
        await _siteA.Inbox.Task.WaitAsync(Arrival, TestContext.Current.CancellationToken);
    }

    // ---- harness ---------------------------------------------------------------------------------

    private static MeshGrpcCommandDialer Dialer(IMeshPskProvider? psk = null) =>
        new(psk ?? new FixedPskProvider(), Options.Create(GrpcOptions()),
            NullLogger<MeshGrpcCommandDialer>.Instance);

    private static MeshTransportOptions GrpcOptions() => new()
    {
        Mode = MeshTransportOptions.ModeGrpc,
        ContactRefreshSeconds = 1,
        DispatchTimeoutSeconds = 10,
    };

    /// <summary>
    ///     Hands out the per-cluster key each site node's interceptor expects. Stands in for the real
    ///     <c>MeshPskProvider</c>, whose own subject (secret-store lookup, caching, the known-cluster
    ///     gate) is <c>MeshPskProviderTests</c>' and would add a KEK and a SQLite file to a class
    ///     whose subject is the wire.
    /// </summary>
    private sealed class FixedPskProvider : IMeshPskProvider
    {
        private readonly string? _override;

        public FixedPskProvider(string? overrideKey = null) => _override = overrideKey;

        public bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk)
        {
            psk = _override ?? KeyFor(clusterId);
            return psk is not null;
        }

        public bool Matches(string clusterId, string presented) =>
            string.Equals(presented, KeyFor(clusterId), StringComparison.Ordinal);

        public void SetKnownClusters(IReadOnlyCollection<string> clusterIds) { }

        public void Invalidate(string clusterId) { }

        public void InvalidateAll() { }

        private static string? KeyFor(string clusterId) => clusterId switch
        {
            "SITE-A" => SiteAKey,
            "SITE-B" => SiteBKey,
            _ => null,
        };
    }

    private static IDbContextFactory<OtOpcUaConfigDbContext> SeedTwoClusterDb(SiteNode a, SiteNode b)
    {
        var factory = new InMemoryConfigDbFactory(Guid.NewGuid().ToString("N"));
        using var db = factory.CreateDbContext();
        db.ClusterNodes.Add(NodeRow("SITE-A", "node-a", a.Port));
        db.ClusterNodes.Add(NodeRow("SITE-B", "node-b", b.Port));
        db.SaveChanges();
        return factory;
    }

    private static ClusterNode NodeRow(string clusterId, string nodeId, int grpcPort) => new()
    {
        NodeId = nodeId,
        ClusterId = clusterId,
        Host = "127.0.0.1",
        AkkaPort = 4053,
        GrpcPort = grpcPort,
        ApplicationUri = $"urn:{nodeId}",
        Enabled = true,
        MaintenanceMode = false,
        CreatedBy = "test",
    };

    private sealed class InMemoryConfigDbFactory(string dbName)
        : IDbContextFactory<OtOpcUaConfigDbContext>
    {
        public OtOpcUaConfigDbContext CreateDbContext()
        {
            var opts = new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
                .UseInMemoryDatabase(dbName)
                .Options;
            return new OtOpcUaConfigDbContext(opts);
        }
    }

    /// <summary>
    ///     One site: its own single-member cluster, its own real <c>NodeCommunicationActor</c>, and a
    ///     real h2c Kestrel listener serving the real <c>NodeCommandGrpcService</c> behind the real
    ///     <c>MeshCommandAuthInterceptor</c> — the production shape of a driver node's command port.
    /// </summary>
    private sealed class SiteNode : IAsyncDisposable
    {
        private WebApplication _server = null!;

        public ActorSystem System { get; private set; } = null!;

        public int Port { get; private set; }

        public string Endpoint => $"http://127.0.0.1:{Port}";

        /// <summary>Completes with the first command re-emitted on this node's EventStream.</summary>
        public TaskCompletionSource<object> Inbox { get; } =
            new(TaskCreationOptions.RunContinuationsAsynchronously);

        public static async Task<SiteNode> StartAsync(string clusterId, string psk)
        {
            var node = new SiteNode { System = StartSelfSeededSystem(), Port = GetFreePort() };

            // The REAL comm actor, under the registry key the gRPC service resolves. Nothing else
            // makes it reachable — there is no receptionist registration any more.
            var comm = node.System.ActorOf(
                NodeCommunicationActor.Props(), MeshPaths.NodeCommunicationName);
            node.System.EventStream.Subscribe(
                node.System.ActorOf(CaptureActor.Props(node.Inbox)), typeof(RestartDriver));

            var registry = new ActorRegistry();
            registry.Register<NodeCommunicationActorKey>(comm);

            var builder = WebApplication.CreateBuilder();
            builder.Environment.ApplicationName = typeof(MeshGrpcBoundaryTests).Assembly.GetName().Name!;
            builder.Logging.ClearProviders();
            builder.Services.AddSingleton(registry);
            builder.Services.Configure<MeshTransportOptions>(o =>
            {
                o.Mode = MeshTransportOptions.ModeGrpc;
                o.GrpcPsk = psk;
            });
            builder.Services.AddGrpc(o => o.Interceptors.Add<MeshCommandAuthInterceptor>());
            builder.WebHost.ConfigureKestrel(k =>
                k.ListenAnyIP(node.Port, o => o.Protocols = HttpProtocols.Http2));

            node._server = builder.Build();
            node._server.MapGrpcService<NodeCommandGrpcService>();
            await node._server.StartAsync(TestContext.Current.CancellationToken);

            _ = clusterId; // Named for readability at the call site; the key IS the cluster boundary.
            return node;
        }

        public async ValueTask DisposeAsync()
        {
            await _server.StopAsync(TimeSpan.FromSeconds(10));
            await _server.DisposeAsync();
            await ShutdownAsync(System);
        }
    }

    /// <summary>Starts a single-member cluster on a dynamic remoting port.</summary>
    /// <remarks>
    ///     Port 0 + a post-start self-Join, rather than a seed-nodes list: the address is not known
    ///     until the transport binds, and hard-coding ports makes the test collide with itself under
    ///     parallel execution.
    /// </remarks>
    /// <returns>The started system.</returns>
    private static ActorSystem StartSelfSeededSystem()
    {
        var config = ConfigurationFactory.ParseString(
                "akka.remote.dot-netty.tcp.hostname = \"127.0.0.1\"\n"
                + "akka.remote.dot-netty.tcp.public-hostname = \"127.0.0.1\"\n"
                + "akka.remote.dot-netty.tcp.port = 0\n"
                + "akka.cluster.seed-nodes = []\n"
                + "akka.loglevel = \"WARNING\"\n")
            .WithFallback(HoconLoader.LoadBaseConfig());

        var system = ActorSystem.Create(SharedSystemName, config);
        var cluster = AkkaCluster.Get(system);
        cluster.Join(cluster.SelfAddress);
        return system;
    }

    private static async Task AwaitSingleMemberUpAsync(ActorSystem system)
    {
        var cluster = AkkaCluster.Get(system);
        var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
        while (DateTime.UtcNow < deadline)
        {
            if (cluster.State.Members.Any(m => m.Status == MemberStatus.Up)) return;
            await Task.Delay(100, TestContext.Current.CancellationToken);
        }

        cluster.State.Members.Any(m => m.Status == MemberStatus.Up).ShouldBeTrue(
            $"{cluster.SelfAddress} never reached Up as its own seed");
    }

    private static async Task ShutdownAsync(ActorSystem? system)
    {
        if (system is null) return;
        await system.Terminate().WaitAsync(TimeSpan.FromSeconds(15));
    }

    private static int GetFreePort()
    {
        using var listener = new System.Net.Sockets.TcpListener(IPAddress.Loopback, 0);
        listener.Start();
        var port = ((IPEndPoint)listener.LocalEndpoint).Port;
        listener.Stop();
        return port;
    }

    /// <summary>Completes an inbox with the first message it receives.</summary>
    private sealed class CaptureActor : ReceiveActor
    {
        public static Props Props(TaskCompletionSource<object> sink) =>
            Akka.Actor.Props.Create(() => new CaptureActor(sink));

        public CaptureActor(TaskCompletionSource<object> sink) => ReceiveAny(m => sink.TrySetResult(m));
    }
}