using System.Collections.Concurrent;
using Akka.Actor;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
using ZB.MOM.WW.ScadaBridge.Communication;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
///
/// T1A.3: + over a real
/// gRPC stack (two in-process central nodes, the real
/// and ). Proves
/// the sticky failover/failback policy, the PSK + site-header attachment, the per-call deadline,
/// and — the hard rule — no cross-node retry on DeadlineExceeded.
///
///
/// A "down" node is modelled by a that throws before reaching the
/// TestServer, so BOTH the unary call and the failback Heartbeat probe see it as
/// Unavailable — the honest shape of a refused connection, and the only class the transport
/// fails over on. Readiness is always set, so a node that is "up" answers everything.
///
public class GrpcCentralTransportTests : IAsyncLifetime
{
private const string SiteA = "site-a";
private const string SiteAKey = "site-a-preshared-key";
private const string EndpointA = "http://central-a/";
private const string EndpointB = "http://central-b/";
private ActorSystem _system = null!;
private CentralNode _nodeA = null!;
private CentralNode _nodeB = null!;
///
public async Task InitializeAsync()
{
_system = ActorSystem.Create("grpc-central-transport-test");
_nodeA = await CentralNode.StartAsync(_system, "A", SiteA, SiteAKey, repliesToSubmit: true);
_nodeB = await CentralNode.StartAsync(_system, "B", SiteA, SiteAKey, repliesToSubmit: true);
}
///
public async Task DisposeAsync()
{
await _nodeA.DisposeAsync();
await _nodeB.DisposeAsync();
await _system.Terminate();
}
private CentralChannelProvider NewProvider(string? pskKey = SiteAKey) => new(
new[] { EndpointA, EndpointB },
new FixedPskProvider(pskKey),
SiteA,
new CommunicationOptions(),
NullLogger.Instance,
handlerFactory: HandlerFor,
probeDeadline: TimeSpan.FromSeconds(2),
backoffBase: TimeSpan.FromMilliseconds(50),
backoffCap: TimeSpan.FromMilliseconds(200));
private HttpMessageHandler HandlerFor(string endpoint) => endpoint == EndpointA
? new ToggleHandler(_nodeA.Server.CreateHandler(), () => _nodeA.IsUp)
: new ToggleHandler(_nodeB.Server.CreateHandler(), () => _nodeB.IsUp);
private GrpcCentralTransport NewTransport(CentralChannelProvider provider, CommunicationOptions? options = null)
=> new(provider, options ?? new CommunicationOptions(), NullLogger.Instance);
[Fact]
public async Task HappyPath_ReachesThePreferredNode_AndRoutesTheAckBack()
{
using var provider = NewProvider();
var transport = NewTransport(provider);
var inbox = new Capture(_system);
transport.SubmitNotification(NewSubmit("n1"), inbox.Ref);
var ack = Assert.IsType(inbox.Receive(TimeSpan.FromSeconds(5)));
Assert.True(ack.Accepted);
Assert.Equal("n1", ack.NotificationId);
Assert.Equal(0, provider.CurrentIndex); // stayed on preferred
Assert.Equal(1, _nodeA.SubmitCount);
Assert.Equal(0, _nodeB.SubmitCount);
}
[Fact]
public async Task Sticky_StaysOnThePreferredNode_WhileHealthy()
{
using var provider = NewProvider();
var transport = NewTransport(provider);
for (var i = 0; i < 4; i++)
{
var inbox = new Capture(_system);
transport.SubmitNotification(NewSubmit($"n{i}"), inbox.Ref);
Assert.IsType(inbox.Receive(TimeSpan.FromSeconds(5)));
}
Assert.Equal(0, provider.CurrentIndex);
Assert.Equal(4, _nodeA.SubmitCount);
Assert.Equal(0, _nodeB.SubmitCount);
}
[Fact]
public async Task Failover_FlipsToThePeer_WhenThePreferredIsUnavailable()
{
using var provider = NewProvider();
var transport = NewTransport(provider);
_nodeA.IsUp = false; // preferred refuses connections
var inbox = new Capture(_system);
transport.SubmitNotification(NewSubmit("n1"), inbox.Ref);
var ack = Assert.IsType(inbox.Receive(TimeSpan.FromSeconds(5)));
Assert.True(ack.Accepted);
Assert.Equal(1, provider.CurrentIndex); // flipped to the peer
Assert.Equal(0, _nodeA.SubmitCount);
Assert.Equal(1, _nodeB.SubmitCount);
}
[Fact]
public async Task Failback_ReturnsToThePreferred_OnceItIsReachableAgain()
{
using var provider = NewProvider();
var transport = NewTransport(provider);
// Take the preferred down and drive one call so we flip to the peer + arm the failback probe.
_nodeA.IsUp = false;
var inbox = new Capture(_system);
transport.SubmitNotification(NewSubmit("n1"), inbox.Ref);
Assert.IsType(inbox.Receive(TimeSpan.FromSeconds(5)));
Assert.Equal(1, provider.CurrentIndex);
// Bring the preferred back; the background probe should fail us back within a few backoffs.
_nodeA.IsUp = true;
await WaitUntil(() => provider.CurrentIndex == 0, TimeSpan.FromSeconds(5));
Assert.Equal(0, provider.CurrentIndex);
// New calls resume on the preferred node.
var inbox2 = new Capture(_system);
transport.SubmitNotification(NewSubmit("n2"), inbox2.Ref);
Assert.IsType(inbox2.Receive(TimeSpan.FromSeconds(5)));
Assert.True(_nodeA.SubmitCount >= 1);
}
[Fact]
public async Task PskAndSiteHeader_AreAttached_SoTheGatedCallReachesTheService()
{
// The service is gated by CentralControlAuthInterceptor; a call that reaches it (and gets
// Accepted) proves both the bearer PSK and the x-scadabridge-site header were attached.
using var provider = NewProvider(pskKey: SiteAKey);
var transport = NewTransport(provider);
var inbox = new Capture(_system);
transport.SubmitNotification(NewSubmit("n1"), inbox.Ref);
var ack = Assert.IsType(inbox.Receive(TimeSpan.FromSeconds(5)));
Assert.True(ack.Accepted);
}
[Fact]
public async Task WrongPsk_IsRejected_AndNotRetriedOnThePeer()
{
// PermissionDenied is not a connect failure — the transport surfaces it as a transient
// Status.Failure without flipping to the peer.
using var provider = NewProvider(pskKey: "the-wrong-key");
var transport = NewTransport(provider);
var inbox = new Capture(_system);
transport.SubmitNotification(NewSubmit("n1"), inbox.Ref);
Assert.IsType(inbox.Receive(TimeSpan.FromSeconds(5)));
Assert.Equal(0, provider.CurrentIndex); // no flip
Assert.Equal(0, _nodeB.SubmitCount); // peer never tried
}
[Fact]
public async Task DeadlineExceeded_IsNotRetriedOnThePeer()
{
// THE hard rule. Node A is UP but never replies, so the call deadlines. The transport must
// surface Status.Failure and must NOT try node B (the call may already have executed).
_nodeA.SetBlackHole();
var shortDeadline = new CommunicationOptions { NotificationForwardTimeout = TimeSpan.FromMilliseconds(300) };
using var provider = NewProvider();
var transport = NewTransport(provider, shortDeadline);
var inbox = new Capture(_system);
transport.SubmitNotification(NewSubmit("n1"), inbox.Ref);
// A per-call deadline is applied (the call returns fast instead of hanging on the black hole).
Assert.IsType(inbox.Receive(TimeSpan.FromSeconds(5)));
Assert.Equal(0, provider.CurrentIndex); // no failover on a deadline
Assert.Equal(0, _nodeB.SubmitCount); // peer never tried
}
private static NotificationSubmit NewSubmit(string id) => new(
NotificationId: id,
ListName: "ops",
Subject: "s",
Body: "b",
SourceSiteId: SiteA,
SourceInstanceId: null,
SourceScript: null,
SiteEnqueuedAt: DateTimeOffset.UtcNow);
private static async Task WaitUntil(Func condition, TimeSpan timeout)
{
var deadline = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < deadline)
{
if (condition())
{
return;
}
await Task.Delay(25);
}
}
///
/// A raw message sink used as the transport's replyTo. Unlike Akka's Inbox, which
/// rethrows a 's cause on receive, this captures every message
/// verbatim so a test can assert on the itself.
///
private sealed class Capture
{
private readonly BlockingCollection