using System.Collections.Concurrent; using Akka.Actor; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.TestHost; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification; using ZB.MOM.WW.ScadaBridge.Communication; using ZB.MOM.WW.ScadaBridge.Communication.Grpc; namespace ZB.MOM.WW.ScadaBridge.Host.Tests; /// /// T1A.3: + over a real /// gRPC stack (two in-process central nodes, the real /// and ). Proves /// the sticky failover/failback policy, the PSK + site-header attachment, the per-call deadline, /// and — the hard rule — no cross-node retry on DeadlineExceeded. /// /// /// A "down" node is modelled by a that throws before reaching the /// TestServer, so BOTH the unary call and the failback Heartbeat probe see it as /// Unavailable — the honest shape of a refused connection, and the only class the transport /// fails over on. Readiness is always set, so a node that is "up" answers everything. /// public class GrpcCentralTransportTests : IAsyncLifetime { private const string SiteA = "site-a"; private const string SiteAKey = "site-a-preshared-key"; private const string EndpointA = "http://central-a/"; private const string EndpointB = "http://central-b/"; private ActorSystem _system = null!; private CentralNode _nodeA = null!; private CentralNode _nodeB = null!; /// public async Task InitializeAsync() { _system = ActorSystem.Create("grpc-central-transport-test"); _nodeA = await CentralNode.StartAsync(_system, "A", SiteA, SiteAKey, repliesToSubmit: true); _nodeB = await CentralNode.StartAsync(_system, "B", SiteA, SiteAKey, repliesToSubmit: true); } /// public async Task DisposeAsync() { await _nodeA.DisposeAsync(); await _nodeB.DisposeAsync(); await _system.Terminate(); } private CentralChannelProvider NewProvider(string? pskKey = SiteAKey) => new( new[] { EndpointA, EndpointB }, new FixedPskProvider(pskKey), SiteA, new CommunicationOptions(), NullLogger.Instance, handlerFactory: HandlerFor, probeDeadline: TimeSpan.FromSeconds(2), backoffBase: TimeSpan.FromMilliseconds(50), backoffCap: TimeSpan.FromMilliseconds(200)); private HttpMessageHandler HandlerFor(string endpoint) => endpoint == EndpointA ? new ToggleHandler(_nodeA.Server.CreateHandler(), () => _nodeA.IsUp) : new ToggleHandler(_nodeB.Server.CreateHandler(), () => _nodeB.IsUp); private GrpcCentralTransport NewTransport(CentralChannelProvider provider, CommunicationOptions? options = null) => new(provider, options ?? new CommunicationOptions(), NullLogger.Instance); [Fact] public async Task HappyPath_ReachesThePreferredNode_AndRoutesTheAckBack() { using var provider = NewProvider(); var transport = NewTransport(provider); var inbox = new Capture(_system); transport.SubmitNotification(NewSubmit("n1"), inbox.Ref); var ack = Assert.IsType(inbox.Receive(TimeSpan.FromSeconds(5))); Assert.True(ack.Accepted); Assert.Equal("n1", ack.NotificationId); Assert.Equal(0, provider.CurrentIndex); // stayed on preferred Assert.Equal(1, _nodeA.SubmitCount); Assert.Equal(0, _nodeB.SubmitCount); } [Fact] public async Task Sticky_StaysOnThePreferredNode_WhileHealthy() { using var provider = NewProvider(); var transport = NewTransport(provider); for (var i = 0; i < 4; i++) { var inbox = new Capture(_system); transport.SubmitNotification(NewSubmit($"n{i}"), inbox.Ref); Assert.IsType(inbox.Receive(TimeSpan.FromSeconds(5))); } Assert.Equal(0, provider.CurrentIndex); Assert.Equal(4, _nodeA.SubmitCount); Assert.Equal(0, _nodeB.SubmitCount); } [Fact] public async Task Failover_FlipsToThePeer_WhenThePreferredIsUnavailable() { using var provider = NewProvider(); var transport = NewTransport(provider); _nodeA.IsUp = false; // preferred refuses connections var inbox = new Capture(_system); transport.SubmitNotification(NewSubmit("n1"), inbox.Ref); var ack = Assert.IsType(inbox.Receive(TimeSpan.FromSeconds(5))); Assert.True(ack.Accepted); Assert.Equal(1, provider.CurrentIndex); // flipped to the peer Assert.Equal(0, _nodeA.SubmitCount); Assert.Equal(1, _nodeB.SubmitCount); } [Fact] public async Task Failback_ReturnsToThePreferred_OnceItIsReachableAgain() { using var provider = NewProvider(); var transport = NewTransport(provider); // Take the preferred down and drive one call so we flip to the peer + arm the failback probe. _nodeA.IsUp = false; var inbox = new Capture(_system); transport.SubmitNotification(NewSubmit("n1"), inbox.Ref); Assert.IsType(inbox.Receive(TimeSpan.FromSeconds(5))); Assert.Equal(1, provider.CurrentIndex); // Bring the preferred back; the background probe should fail us back within a few backoffs. _nodeA.IsUp = true; await WaitUntil(() => provider.CurrentIndex == 0, TimeSpan.FromSeconds(5)); Assert.Equal(0, provider.CurrentIndex); // New calls resume on the preferred node. var inbox2 = new Capture(_system); transport.SubmitNotification(NewSubmit("n2"), inbox2.Ref); Assert.IsType(inbox2.Receive(TimeSpan.FromSeconds(5))); Assert.True(_nodeA.SubmitCount >= 1); } [Fact] public async Task PskAndSiteHeader_AreAttached_SoTheGatedCallReachesTheService() { // The service is gated by CentralControlAuthInterceptor; a call that reaches it (and gets // Accepted) proves both the bearer PSK and the x-scadabridge-site header were attached. using var provider = NewProvider(pskKey: SiteAKey); var transport = NewTransport(provider); var inbox = new Capture(_system); transport.SubmitNotification(NewSubmit("n1"), inbox.Ref); var ack = Assert.IsType(inbox.Receive(TimeSpan.FromSeconds(5))); Assert.True(ack.Accepted); } [Fact] public async Task WrongPsk_IsRejected_AndNotRetriedOnThePeer() { // PermissionDenied is not a connect failure — the transport surfaces it as a transient // Status.Failure without flipping to the peer. using var provider = NewProvider(pskKey: "the-wrong-key"); var transport = NewTransport(provider); var inbox = new Capture(_system); transport.SubmitNotification(NewSubmit("n1"), inbox.Ref); Assert.IsType(inbox.Receive(TimeSpan.FromSeconds(5))); Assert.Equal(0, provider.CurrentIndex); // no flip Assert.Equal(0, _nodeB.SubmitCount); // peer never tried } [Fact] public async Task DeadlineExceeded_IsNotRetriedOnThePeer() { // THE hard rule. Node A is UP but never replies, so the call deadlines. The transport must // surface Status.Failure and must NOT try node B (the call may already have executed). _nodeA.SetBlackHole(); var shortDeadline = new CommunicationOptions { NotificationForwardTimeout = TimeSpan.FromMilliseconds(300) }; using var provider = NewProvider(); var transport = NewTransport(provider, shortDeadline); var inbox = new Capture(_system); transport.SubmitNotification(NewSubmit("n1"), inbox.Ref); // A per-call deadline is applied (the call returns fast instead of hanging on the black hole). Assert.IsType(inbox.Receive(TimeSpan.FromSeconds(5))); Assert.Equal(0, provider.CurrentIndex); // no failover on a deadline Assert.Equal(0, _nodeB.SubmitCount); // peer never tried } private static NotificationSubmit NewSubmit(string id) => new( NotificationId: id, ListName: "ops", Subject: "s", Body: "b", SourceSiteId: SiteA, SourceInstanceId: null, SourceScript: null, SiteEnqueuedAt: DateTimeOffset.UtcNow); private static async Task WaitUntil(Func condition, TimeSpan timeout) { var deadline = DateTime.UtcNow + timeout; while (DateTime.UtcNow < deadline) { if (condition()) { return; } await Task.Delay(25); } } /// /// A raw message sink used as the transport's replyTo. Unlike Akka's Inbox, which /// rethrows a 's cause on receive, this captures every message /// verbatim so a test can assert on the itself. /// private sealed class Capture { private readonly BlockingCollection _messages = new(); public Capture(ActorSystem system) { Ref = system.ActorOf(Props.Create(() => new CaptureActor(_messages))); } public IActorRef Ref { get; } public object Receive(TimeSpan timeout) => _messages.TryTake(out var message, timeout) ? message : throw new TimeoutException("No message captured within the timeout."); private sealed class CaptureActor : ReceiveActor { public CaptureActor(BlockingCollection messages) => ReceiveAny(messages.Add); } } /// A gRPC channel handler that throws (a refused connection) while its node is "down". private sealed class ToggleHandler : DelegatingHandler { private readonly Func _isUp; public ToggleHandler(HttpMessageHandler inner, Func isUp) { InnerHandler = inner; _isUp = isUp; } protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { if (!_isUp()) { throw new HttpRequestException("simulated central node down"); } return await base.SendAsync(request, cancellationToken).ConfigureAwait(false); } } private sealed class FixedPskProvider(string? key) : ISitePskProvider { public ValueTask GetAsync(string siteId, CancellationToken ct) => key is null ? throw new InvalidOperationException("no key") : new ValueTask(key); public void Invalidate(string siteId) { } } /// One in-process central node: TestServer + real service/interceptor + a stub actor. private sealed class CentralNode : IAsyncDisposable { private IHost _host = null!; private IActorRef _stub = null!; private readonly StubCounters _counters = new(); public TestServer Server { get; private set; } = null!; public volatile bool IsUp = true; public int SubmitCount => _counters.Submits; public static async Task StartAsync( ActorSystem system, string label, string site, string key, bool repliesToSubmit) { var node = new CentralNode(); node._stub = system.ActorOf( Props.Create(() => new StubCentralActor(node._counters, repliesToSubmit)), $"stub-{label}"); var service = new CentralControlGrpcService( NullLogger.Instance, Options.Create(new CommunicationOptions())); service.SetReady(node._stub); var psk = new MapPskProvider(new Dictionary { [site] = key }); node._host = await new HostBuilder() .ConfigureWebHost(web => web .UseTestServer() .ConfigureServices(services => { services.AddGrpc(o => o.Interceptors.Add()); services.AddSingleton(psk); services.AddSingleton(service); }) .Configure(app => { app.UseRouting(); app.UseEndpoints(e => e.MapGrpcService()); })) .StartAsync(); node.Server = node._host.GetTestServer(); return node; } /// Switches the node's actor to a black hole that counts but never replies. public void SetBlackHole() => _counters.BlackHole = true; public async ValueTask DisposeAsync() { await _host.StopAsync(); _host.Dispose(); } private sealed class StubCounters { private int _submits; public int Submits => Volatile.Read(ref _submits); public void IncrementSubmits() => Interlocked.Increment(ref _submits); public volatile bool BlackHole; } private sealed class StubCentralActor : ReceiveActor { public StubCentralActor(StubCounters counters, bool repliesToSubmit) { Receive(msg => { counters.IncrementSubmits(); if (repliesToSubmit && !counters.BlackHole) { Sender.Tell(new NotificationSubmitAck(msg.NotificationId, Accepted: true, Error: null)); } }); // Heartbeat lands here as a Tell (the failback probe); ignore it, no reply expected. ReceiveAny(_ => { }); } } private sealed class MapPskProvider(IReadOnlyDictionary keys) : ISitePskProvider { public ValueTask GetAsync(string siteId, CancellationToken ct) => keys.TryGetValue(siteId, out var key) ? new ValueTask(key) : throw new InvalidOperationException($"no key for '{siteId}'"); public void Invalidate(string siteId) { } } } }