using System.Diagnostics.CodeAnalysis;
using System.Net;
using Akka.Actor;
using Akka.Cluster;
using Akka.Configuration;
using Akka.Hosting;
using Grpc.Core;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Server.Kestrel.Core;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
using ZB.MOM.WW.OtOpcUa.Host.Grpc;
using ZB.MOM.WW.OtOpcUa.Runtime;
using ZB.MOM.WW.OtOpcUa.Runtime.Communication;
using AkkaCluster = Akka.Cluster.Cluster;
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
///
/// Drives the mesh command plane across two genuinely separate s over
/// a real gRPC wire — the only test in the repo where the central↔node boundary carries a
/// message off-process.
///
///
///
/// Replaces MeshClusterClientBoundaryTests, which did the same job for the Akka
/// ClusterClient transport that this program deleted. The reason it exists is unchanged:
/// a fake dialer tests actor logic and cannot fail for any transport reason — not a
/// service that was never mapped, not an interceptor that refuses the call, not a
/// serialization break. Those are the failure modes this class exercises, end to end
/// through the REAL CentralCommunicationActor, MeshGrpcCommandDialer,
/// MeshCommandAuthInterceptor, NodeCommandGrpcService and
/// NodeCommunicationActor.
///
///
/// Three clusters, one ActorSystem name, joined ONLY by gRPC. The shared name is kept
/// from the predecessor so the shape is comparable; each system seeds itself and none lists
/// another, and
///
/// asserts the separation outright. That assertion is the whole point of the program — the
/// cross-WAN Akka association is gone — so it is kept as a test rather than left to a live
/// gate: if the three systems ever gossiped into one mesh, every delivery below could be
/// plain local messaging and the class would be testing nothing.
///
///
/// Per-cluster keys, not one fleet key. Each site node's interceptor is configured
/// with its OWN MeshTransport:GrpcPsk and central's provider hands the dialer the key
/// for the cluster it is dialling. A fan that presented the wrong cluster's key would be
/// refused at the far side, so the happy path below also pins that the dialer selects by
/// ClusterId.
///
///
public sealed class MeshGrpcBoundaryTests : IAsyncLifetime
{
private const string SharedSystemName = "otopcua";
private const string SiteAKey = "psk-site-a";
private const string SiteBKey = "psk-site-b";
private static readonly TimeSpan Arrival = TimeSpan.FromSeconds(30);
///
/// How long the falsifiability control waits before concluding nothing arrived. Deliberately
/// shorter than but far longer than a successful delivery takes in
/// practice, so a pass means "the boundary refused it", not "the clock ran out first".
///
private static readonly TimeSpan Silence = TimeSpan.FromSeconds(10);
private ActorSystem _central = null!;
private SiteNode _siteA = null!;
private SiteNode _siteB = null!;
///
public async ValueTask InitializeAsync()
{
_central = StartSelfSeededSystem();
_siteA = await SiteNode.StartAsync("SITE-A", SiteAKey);
_siteB = await SiteNode.StartAsync("SITE-B", SiteBKey);
await Task.WhenAll(
AwaitSingleMemberUpAsync(_central),
AwaitSingleMemberUpAsync(_siteA.System),
AwaitSingleMemberUpAsync(_siteB.System));
}
///
public async ValueTask DisposeAsync()
{
await _siteA.DisposeAsync();
await _siteB.DisposeAsync();
await ShutdownAsync(_central);
}
[Fact]
public void No_cross_mesh_Akka_association_forms_between_central_and_either_site()
{
// THE assertion of the ClusterClient→gRPC program, kept as a test rather than a live-gate
// note. Three systems sharing a name in one process are one gossip accident away from being
// one cluster, at which point the gRPC boundary is no longer the thing under test — and,
// more importantly, the cross-WAN Akka association this program removed would be back.
var central = AkkaCluster.Get(_central);
var siteA = AkkaCluster.Get(_siteA.System);
var siteB = AkkaCluster.Get(_siteB.System);
central.State.Members.Count.ShouldBe(1,
"central must be its own single-member cluster; commands cross to a site over gRPC only");
siteA.State.Members.Count.ShouldBe(1, "site-A must be its own single-member cluster");
siteB.State.Members.Count.ShouldBe(1, "site-B must be its own single-member cluster");
// Stated separately from the counts: a count of 1 on each side would also hold in the instant
// before a join completed, whereas naming the addresses says no side ever learned another's.
foreach (var (name, cluster) in new[]
{
("central", central), ("site-A", siteA), ("site-B", siteB),
})
{
var others = new[] { central.SelfAddress, siteA.SelfAddress, siteB.SelfAddress }
.Where(a => a != cluster.SelfAddress);
foreach (var other in others)
{
cluster.State.Members.ShouldNotContain(
m => m.Address == other,
$"{name} must not have {other} as a cluster member — the cross-mesh Akka "
+ "association is exactly what the gRPC command plane replaced");
}
}
}
[Fact]
public async Task One_dispatch_through_central_reaches_a_node_in_every_cluster_mesh()
{
// The direct counterpart of the deleted ClusterClient fan-out test, at the transport level and
// driving the REAL CentralCommunicationActor. Two genuinely separate site meshes, each with
// its own gRPC command endpoint and its own per-cluster key; one MeshCommand must reach BOTH.
var dbFactory = SeedTwoClusterDb(_siteA, _siteB);
var actor = _central.ActorOf(CentralCommunicationActor.Props(
dbFactory,
GrpcOptions(),
() => null,
() => Dialer(),
() => new FixedPskProvider()));
var command = new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid());
var deadline = DateTime.UtcNow + Arrival;
while (DateTime.UtcNow < deadline
&& !(_siteA.Inbox.Task.IsCompleted && _siteB.Inbox.Task.IsCompleted))
{
// Re-sent rather than sent once: the actor's first contact load is a piped DB read, and a
// command told before it lands is dropped as "no targets" by design. Re-sending weakens
// nothing — the wrong-key control below re-sends on the same schedule and must still
// receive nothing.
actor.Tell(new MeshCommand("deployments", command));
await Task.Delay(250, TestContext.Current.CancellationToken);
}
_siteA.Inbox.Task.IsCompleted.ShouldBeTrue(
"site-A's mesh must receive the dispatch on its own gRPC command endpoint");
_siteB.Inbox.Task.IsCompleted.ShouldBeTrue(
"site-B's mesh must receive the SAME dispatch on its own endpoint, under its own "
+ "per-cluster key — a fan that stopped at the first cluster would leave site-B silently "
+ "on its old configuration while the deployment sealed green");
(await _siteA.Inbox.Task).ShouldBeOfType().DriverInstanceId.ShouldBe("driver-1");
(await _siteB.Inbox.Task).ShouldBeOfType().DriverInstanceId.ShouldBe("driver-1");
}
[Fact]
public async Task Falsifiability_control_a_dial_carrying_the_wrong_cluster_key_delivers_nothing()
{
// Without this, the test above cannot distinguish "the gRPC boundary carried the command"
// from "something else in the process delivered it". Same wire, same service, same comm
// actor — and with a key site-A's interceptor does not accept, it must NOT arrive.
var envelope = MeshTransportDtoMapper.ToEnvelope(
new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid()));
var dialer = Dialer(new FixedPskProvider("not-the-key"));
var deadline = DateTime.UtcNow + Silence;
while (DateTime.UtcNow < deadline && !_siteA.Inbox.Task.IsCompleted)
{
try
{
await dialer.DispatchAsync(
_siteA.Endpoint, envelope, "SITE-A", TestContext.Current.CancellationToken);
}
catch (RpcException)
{
// Expected: the node's MeshCommandAuthInterceptor refuses the bearer. The assertion
// is what did NOT arrive, so the rejection itself is swallowed rather than asserted
// on here — MeshCommandAuthInterceptorTests owns the status-code contract.
}
await Task.Delay(500, TestContext.Current.CancellationToken);
}
_siteA.Inbox.Task.IsCompleted.ShouldBeFalse(
"delivery survived presenting the wrong per-cluster key, so the positive test above is "
+ "not measuring the authenticated gRPC boundary at all");
// Same node, same dialer shape, right key: proves the silence above was the auth gate rather
// than a dead endpoint or a misaddressed call.
await Dialer().DispatchAsync(
_siteA.Endpoint, envelope, "SITE-A", TestContext.Current.CancellationToken);
await _siteA.Inbox.Task.WaitAsync(Arrival, TestContext.Current.CancellationToken);
}
// ---- harness ---------------------------------------------------------------------------------
private static MeshGrpcCommandDialer Dialer(IMeshPskProvider? psk = null) =>
new(psk ?? new FixedPskProvider(), Options.Create(GrpcOptions()),
NullLogger.Instance);
private static MeshTransportOptions GrpcOptions() => new()
{
Mode = MeshTransportOptions.ModeGrpc,
ContactRefreshSeconds = 1,
DispatchTimeoutSeconds = 10,
};
///
/// Hands out the per-cluster key each site node's interceptor expects. Stands in for the real
/// MeshPskProvider, whose own subject (secret-store lookup, caching, the known-cluster
/// gate) is MeshPskProviderTests' and would add a KEK and a SQLite file to a class
/// whose subject is the wire.
///
private sealed class FixedPskProvider : IMeshPskProvider
{
private readonly string? _override;
public FixedPskProvider(string? overrideKey = null) => _override = overrideKey;
public bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk)
{
psk = _override ?? KeyFor(clusterId);
return psk is not null;
}
public bool Matches(string clusterId, string presented) =>
string.Equals(presented, KeyFor(clusterId), StringComparison.Ordinal);
public void SetKnownClusters(IReadOnlyCollection clusterIds) { }
public void Invalidate(string clusterId) { }
public void InvalidateAll() { }
private static string? KeyFor(string clusterId) => clusterId switch
{
"SITE-A" => SiteAKey,
"SITE-B" => SiteBKey,
_ => null,
};
}
private static IDbContextFactory SeedTwoClusterDb(SiteNode a, SiteNode b)
{
var factory = new InMemoryConfigDbFactory(Guid.NewGuid().ToString("N"));
using var db = factory.CreateDbContext();
db.ClusterNodes.Add(NodeRow("SITE-A", "node-a", a.Port));
db.ClusterNodes.Add(NodeRow("SITE-B", "node-b", b.Port));
db.SaveChanges();
return factory;
}
private static ClusterNode NodeRow(string clusterId, string nodeId, int grpcPort) => new()
{
NodeId = nodeId,
ClusterId = clusterId,
Host = "127.0.0.1",
AkkaPort = 4053,
GrpcPort = grpcPort,
ApplicationUri = $"urn:{nodeId}",
Enabled = true,
MaintenanceMode = false,
CreatedBy = "test",
};
private sealed class InMemoryConfigDbFactory(string dbName)
: IDbContextFactory
{
public OtOpcUaConfigDbContext CreateDbContext()
{
var opts = new DbContextOptionsBuilder()
.UseInMemoryDatabase(dbName)
.Options;
return new OtOpcUaConfigDbContext(opts);
}
}
///
/// One site: its own single-member cluster, its own real NodeCommunicationActor, and a
/// real h2c Kestrel listener serving the real NodeCommandGrpcService behind the real
/// MeshCommandAuthInterceptor — the production shape of a driver node's command port.
///
private sealed class SiteNode : IAsyncDisposable
{
private WebApplication _server = null!;
public ActorSystem System { get; private set; } = null!;
public int Port { get; private set; }
public string Endpoint => $"http://127.0.0.1:{Port}";
/// Completes with the first command re-emitted on this node's EventStream.
public TaskCompletionSource