using System.Diagnostics.CodeAnalysis; using System.Net; using Akka.Actor; using Akka.Cluster; using Akka.Configuration; using Akka.Hosting; using Grpc.Core; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Server.Kestrel.Core; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; using ZB.MOM.WW.OtOpcUa.Host.Configuration; using ZB.MOM.WW.OtOpcUa.Host.Grpc; using ZB.MOM.WW.OtOpcUa.Runtime; using ZB.MOM.WW.OtOpcUa.Runtime.Communication; using AkkaCluster = Akka.Cluster.Cluster; namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; /// /// Drives the mesh command plane across two genuinely separate s over /// a real gRPC wire — the only test in the repo where the central↔node boundary carries a /// message off-process. /// /// /// /// Replaces MeshClusterClientBoundaryTests, which did the same job for the Akka /// ClusterClient transport that this program deleted. The reason it exists is unchanged: /// a fake dialer tests actor logic and cannot fail for any transport reason — not a /// service that was never mapped, not an interceptor that refuses the call, not a /// serialization break. Those are the failure modes this class exercises, end to end /// through the REAL CentralCommunicationActor, MeshGrpcCommandDialer, /// MeshCommandAuthInterceptor, NodeCommandGrpcService and /// NodeCommunicationActor. /// /// /// Three clusters, one ActorSystem name, joined ONLY by gRPC. The shared name is kept /// from the predecessor so the shape is comparable; each system seeds itself and none lists /// another, and /// /// asserts the separation outright. That assertion is the whole point of the program — the /// cross-WAN Akka association is gone — so it is kept as a test rather than left to a live /// gate: if the three systems ever gossiped into one mesh, every delivery below could be /// plain local messaging and the class would be testing nothing. /// /// /// Per-cluster keys, not one fleet key. Each site node's interceptor is configured /// with its OWN MeshTransport:GrpcPsk and central's provider hands the dialer the key /// for the cluster it is dialling. A fan that presented the wrong cluster's key would be /// refused at the far side, so the happy path below also pins that the dialer selects by /// ClusterId. /// /// public sealed class MeshGrpcBoundaryTests : IAsyncLifetime { private const string SharedSystemName = "otopcua"; private const string SiteAKey = "psk-site-a"; private const string SiteBKey = "psk-site-b"; private static readonly TimeSpan Arrival = TimeSpan.FromSeconds(30); /// /// How long the falsifiability control waits before concluding nothing arrived. Deliberately /// shorter than but far longer than a successful delivery takes in /// practice, so a pass means "the boundary refused it", not "the clock ran out first". /// private static readonly TimeSpan Silence = TimeSpan.FromSeconds(10); private ActorSystem _central = null!; private SiteNode _siteA = null!; private SiteNode _siteB = null!; /// public async ValueTask InitializeAsync() { _central = StartSelfSeededSystem(); _siteA = await SiteNode.StartAsync("SITE-A", SiteAKey); _siteB = await SiteNode.StartAsync("SITE-B", SiteBKey); await Task.WhenAll( AwaitSingleMemberUpAsync(_central), AwaitSingleMemberUpAsync(_siteA.System), AwaitSingleMemberUpAsync(_siteB.System)); } /// public async ValueTask DisposeAsync() { await _siteA.DisposeAsync(); await _siteB.DisposeAsync(); await ShutdownAsync(_central); } [Fact] public void No_cross_mesh_Akka_association_forms_between_central_and_either_site() { // THE assertion of the ClusterClient→gRPC program, kept as a test rather than a live-gate // note. Three systems sharing a name in one process are one gossip accident away from being // one cluster, at which point the gRPC boundary is no longer the thing under test — and, // more importantly, the cross-WAN Akka association this program removed would be back. var central = AkkaCluster.Get(_central); var siteA = AkkaCluster.Get(_siteA.System); var siteB = AkkaCluster.Get(_siteB.System); central.State.Members.Count.ShouldBe(1, "central must be its own single-member cluster; commands cross to a site over gRPC only"); siteA.State.Members.Count.ShouldBe(1, "site-A must be its own single-member cluster"); siteB.State.Members.Count.ShouldBe(1, "site-B must be its own single-member cluster"); // Stated separately from the counts: a count of 1 on each side would also hold in the instant // before a join completed, whereas naming the addresses says no side ever learned another's. foreach (var (name, cluster) in new[] { ("central", central), ("site-A", siteA), ("site-B", siteB), }) { var others = new[] { central.SelfAddress, siteA.SelfAddress, siteB.SelfAddress } .Where(a => a != cluster.SelfAddress); foreach (var other in others) { cluster.State.Members.ShouldNotContain( m => m.Address == other, $"{name} must not have {other} as a cluster member — the cross-mesh Akka " + "association is exactly what the gRPC command plane replaced"); } } } [Fact] public async Task One_dispatch_through_central_reaches_a_node_in_every_cluster_mesh() { // The direct counterpart of the deleted ClusterClient fan-out test, at the transport level and // driving the REAL CentralCommunicationActor. Two genuinely separate site meshes, each with // its own gRPC command endpoint and its own per-cluster key; one MeshCommand must reach BOTH. var dbFactory = SeedTwoClusterDb(_siteA, _siteB); var actor = _central.ActorOf(CentralCommunicationActor.Props( dbFactory, GrpcOptions(), () => null, () => Dialer(), () => new FixedPskProvider())); var command = new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid()); var deadline = DateTime.UtcNow + Arrival; while (DateTime.UtcNow < deadline && !(_siteA.Inbox.Task.IsCompleted && _siteB.Inbox.Task.IsCompleted)) { // Re-sent rather than sent once: the actor's first contact load is a piped DB read, and a // command told before it lands is dropped as "no targets" by design. Re-sending weakens // nothing — the wrong-key control below re-sends on the same schedule and must still // receive nothing. actor.Tell(new MeshCommand("deployments", command)); await Task.Delay(250, TestContext.Current.CancellationToken); } _siteA.Inbox.Task.IsCompleted.ShouldBeTrue( "site-A's mesh must receive the dispatch on its own gRPC command endpoint"); _siteB.Inbox.Task.IsCompleted.ShouldBeTrue( "site-B's mesh must receive the SAME dispatch on its own endpoint, under its own " + "per-cluster key — a fan that stopped at the first cluster would leave site-B silently " + "on its old configuration while the deployment sealed green"); (await _siteA.Inbox.Task).ShouldBeOfType().DriverInstanceId.ShouldBe("driver-1"); (await _siteB.Inbox.Task).ShouldBeOfType().DriverInstanceId.ShouldBe("driver-1"); } [Fact] public async Task Falsifiability_control_a_dial_carrying_the_wrong_cluster_key_delivers_nothing() { // Without this, the test above cannot distinguish "the gRPC boundary carried the command" // from "something else in the process delivered it". Same wire, same service, same comm // actor — and with a key site-A's interceptor does not accept, it must NOT arrive. var envelope = MeshTransportDtoMapper.ToEnvelope( new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid())); var dialer = Dialer(new FixedPskProvider("not-the-key")); var deadline = DateTime.UtcNow + Silence; while (DateTime.UtcNow < deadline && !_siteA.Inbox.Task.IsCompleted) { try { await dialer.DispatchAsync( _siteA.Endpoint, envelope, "SITE-A", TestContext.Current.CancellationToken); } catch (RpcException) { // Expected: the node's MeshCommandAuthInterceptor refuses the bearer. The assertion // is what did NOT arrive, so the rejection itself is swallowed rather than asserted // on here — MeshCommandAuthInterceptorTests owns the status-code contract. } await Task.Delay(500, TestContext.Current.CancellationToken); } _siteA.Inbox.Task.IsCompleted.ShouldBeFalse( "delivery survived presenting the wrong per-cluster key, so the positive test above is " + "not measuring the authenticated gRPC boundary at all"); // Same node, same dialer shape, right key: proves the silence above was the auth gate rather // than a dead endpoint or a misaddressed call. await Dialer().DispatchAsync( _siteA.Endpoint, envelope, "SITE-A", TestContext.Current.CancellationToken); await _siteA.Inbox.Task.WaitAsync(Arrival, TestContext.Current.CancellationToken); } // ---- harness --------------------------------------------------------------------------------- private static MeshGrpcCommandDialer Dialer(IMeshPskProvider? psk = null) => new(psk ?? new FixedPskProvider(), Options.Create(GrpcOptions()), NullLogger.Instance); private static MeshTransportOptions GrpcOptions() => new() { Mode = MeshTransportOptions.ModeGrpc, ContactRefreshSeconds = 1, DispatchTimeoutSeconds = 10, }; /// /// Hands out the per-cluster key each site node's interceptor expects. Stands in for the real /// MeshPskProvider, whose own subject (secret-store lookup, caching, the known-cluster /// gate) is MeshPskProviderTests' and would add a KEK and a SQLite file to a class /// whose subject is the wire. /// private sealed class FixedPskProvider : IMeshPskProvider { private readonly string? _override; public FixedPskProvider(string? overrideKey = null) => _override = overrideKey; public bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk) { psk = _override ?? KeyFor(clusterId); return psk is not null; } public bool Matches(string clusterId, string presented) => string.Equals(presented, KeyFor(clusterId), StringComparison.Ordinal); public void SetKnownClusters(IReadOnlyCollection clusterIds) { } public void Invalidate(string clusterId) { } public void InvalidateAll() { } private static string? KeyFor(string clusterId) => clusterId switch { "SITE-A" => SiteAKey, "SITE-B" => SiteBKey, _ => null, }; } private static IDbContextFactory SeedTwoClusterDb(SiteNode a, SiteNode b) { var factory = new InMemoryConfigDbFactory(Guid.NewGuid().ToString("N")); using var db = factory.CreateDbContext(); db.ClusterNodes.Add(NodeRow("SITE-A", "node-a", a.Port)); db.ClusterNodes.Add(NodeRow("SITE-B", "node-b", b.Port)); db.SaveChanges(); return factory; } private static ClusterNode NodeRow(string clusterId, string nodeId, int grpcPort) => new() { NodeId = nodeId, ClusterId = clusterId, Host = "127.0.0.1", AkkaPort = 4053, GrpcPort = grpcPort, ApplicationUri = $"urn:{nodeId}", Enabled = true, MaintenanceMode = false, CreatedBy = "test", }; private sealed class InMemoryConfigDbFactory(string dbName) : IDbContextFactory { public OtOpcUaConfigDbContext CreateDbContext() { var opts = new DbContextOptionsBuilder() .UseInMemoryDatabase(dbName) .Options; return new OtOpcUaConfigDbContext(opts); } } /// /// One site: its own single-member cluster, its own real NodeCommunicationActor, and a /// real h2c Kestrel listener serving the real NodeCommandGrpcService behind the real /// MeshCommandAuthInterceptor — the production shape of a driver node's command port. /// private sealed class SiteNode : IAsyncDisposable { private WebApplication _server = null!; public ActorSystem System { get; private set; } = null!; public int Port { get; private set; } public string Endpoint => $"http://127.0.0.1:{Port}"; /// Completes with the first command re-emitted on this node's EventStream. public TaskCompletionSource Inbox { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public static async Task StartAsync(string clusterId, string psk) { var node = new SiteNode { System = StartSelfSeededSystem(), Port = GetFreePort() }; // The REAL comm actor, under the registry key the gRPC service resolves. Nothing else // makes it reachable — there is no receptionist registration any more. var comm = node.System.ActorOf( NodeCommunicationActor.Props(), MeshPaths.NodeCommunicationName); node.System.EventStream.Subscribe( node.System.ActorOf(CaptureActor.Props(node.Inbox)), typeof(RestartDriver)); var registry = new ActorRegistry(); registry.Register(comm); var builder = WebApplication.CreateBuilder(); builder.Environment.ApplicationName = typeof(MeshGrpcBoundaryTests).Assembly.GetName().Name!; builder.Logging.ClearProviders(); builder.Services.AddSingleton(registry); builder.Services.Configure(o => { o.Mode = MeshTransportOptions.ModeGrpc; o.GrpcPsk = psk; }); builder.Services.AddGrpc(o => o.Interceptors.Add()); builder.WebHost.ConfigureKestrel(k => k.ListenAnyIP(node.Port, o => o.Protocols = HttpProtocols.Http2)); node._server = builder.Build(); node._server.MapGrpcService(); await node._server.StartAsync(TestContext.Current.CancellationToken); _ = clusterId; // Named for readability at the call site; the key IS the cluster boundary. return node; } public async ValueTask DisposeAsync() { await _server.StopAsync(TimeSpan.FromSeconds(10)); await _server.DisposeAsync(); await ShutdownAsync(System); } } /// Starts a single-member cluster on a dynamic remoting port. /// /// Port 0 + a post-start self-Join, rather than a seed-nodes list: the address is not known /// until the transport binds, and hard-coding ports makes the test collide with itself under /// parallel execution. /// /// The started system. private static ActorSystem StartSelfSeededSystem() { var config = ConfigurationFactory.ParseString( "akka.remote.dot-netty.tcp.hostname = \"127.0.0.1\"\n" + "akka.remote.dot-netty.tcp.public-hostname = \"127.0.0.1\"\n" + "akka.remote.dot-netty.tcp.port = 0\n" + "akka.cluster.seed-nodes = []\n" + "akka.loglevel = \"WARNING\"\n") .WithFallback(HoconLoader.LoadBaseConfig()); var system = ActorSystem.Create(SharedSystemName, config); var cluster = AkkaCluster.Get(system); cluster.Join(cluster.SelfAddress); return system; } private static async Task AwaitSingleMemberUpAsync(ActorSystem system) { var cluster = AkkaCluster.Get(system); var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); while (DateTime.UtcNow < deadline) { if (cluster.State.Members.Any(m => m.Status == MemberStatus.Up)) return; await Task.Delay(100, TestContext.Current.CancellationToken); } cluster.State.Members.Any(m => m.Status == MemberStatus.Up).ShouldBeTrue( $"{cluster.SelfAddress} never reached Up as its own seed"); } private static async Task ShutdownAsync(ActorSystem? system) { if (system is null) return; await system.Terminate().WaitAsync(TimeSpan.FromSeconds(15)); } private static int GetFreePort() { using var listener = new System.Net.Sockets.TcpListener(IPAddress.Loopback, 0); listener.Start(); var port = ((IPEndPoint)listener.LocalEndpoint).Port; listener.Stop(); return port; } /// Completes an inbox with the first message it receives. private sealed class CaptureActor : ReceiveActor { public static Props Props(TaskCompletionSource sink) => Akka.Actor.Props.Create(() => new CaptureActor(sink)); public CaptureActor(TaskCompletionSource sink) => ReceiveAny(m => sink.TrySetResult(m)); } }