using Grpc.Core; using Grpc.Net.Client; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.TestHost; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle; using ZB.MOM.WW.ScadaBridge.Communication; using ZB.MOM.WW.ScadaBridge.Communication.Grpc; namespace ZB.MOM.WW.ScadaBridge.Host.Tests; /// /// T1B.3: failover/failback + credential/deadline proof over /// two in-process gRPC s (NodeA preferred, NodeB standby). Exercises the /// real client stack — PSK call credentials, per-call deadline, sticky failover, no-retry on /// , and failback to the preferred node. /// public sealed class GrpcSiteTransportFailoverTests : IAsyncLifetime { private const string SiteId = "site-1"; private const string SiteKey = "the-site-1-key"; private const string EndpointA = "http://node-a"; private const string EndpointB = "http://node-b"; private IHost _hostA = null!; private IHost _hostB = null!; private StubSiteCommandService _stubA = null!; private StubSiteCommandService _stubB = null!; private SitePairChannelProvider _provider = null!; private volatile bool _preferredReachable; /// public async Task InitializeAsync() { (_hostA, _stubA) = await StartServerAsync(); (_hostB, _stubB) = await StartServerAsync(); var handlerA = _hostA.GetTestServer().CreateHandler(); var handlerB = _hostB.GetTestServer().CreateHandler(); _provider = new SitePairChannelProvider( new FixedPskProvider(SiteKey), Options.Create(new CommunicationOptions()), NullLogger.Instance, handlerFactory: endpoint => endpoint == EndpointA ? handlerA : handlerB, reachabilityProbe: (_, _) => Task.FromResult(_preferredReachable)); _provider.UpdateSite(SiteId, EndpointA, EndpointB); } /// public async Task DisposeAsync() { _provider.Dispose(); await _hostA.StopAsync(); await _hostB.StopAsync(); _hostA.Dispose(); _hostB.Dispose(); } private static async Task<(IHost, StubSiteCommandService)> StartServerAsync() { var stub = new StubSiteCommandService(); var host = await new HostBuilder() .ConfigureWebHost(web => web .UseTestServer() .ConfigureServices(services => { services.AddGrpc(); services.AddSingleton(stub); }) .Configure(app => { app.UseRouting(); app.UseEndpoints(e => e.MapGrpcService()); })) .StartAsync(); return (host, stub); } private static LifecycleRequest EnableReq() => SiteCommandDtoMapper.ToLifecycleRequest( new EnableInstanceCommand("cmd-1", "Site1.Pump1", DateTimeOffset.UtcNow)); private Task CallAsync(TimeSpan? deadline = null) => _provider.ExecuteAsync( SiteId, (channel, ct) => { var client = new SiteCommandService.SiteCommandServiceClient(channel); return client.ExecuteLifecycleAsync( EnableReq(), deadline: DateTime.UtcNow + (deadline ?? TimeSpan.FromSeconds(10)), cancellationToken: ct).ResponseAsync; }, CancellationToken.None); [Fact] public async Task HealthyCall_HitsPreferredNodeA_WithPskAndSiteHeaderAndDeadline() { var reply = await CallAsync(); Assert.Equal(LifecycleReply.ReplyOneofCase.InstanceLifecycle, reply.ReplyCase); Assert.Equal(1, _stubA.LifecycleCalls); Assert.Equal(0, _stubB.LifecycleCalls); Assert.Equal($"Bearer {SiteKey}", _stubA.LastAuthHeader); Assert.Equal(SiteId, _stubA.LastSiteHeader); Assert.True(_stubA.DeadlineWasSet, "the client must set a per-call deadline"); Assert.True(_provider.IsOnPreferredNode(SiteId)); } [Fact] public async Task Unavailable_OnNodeA_FailsOverToNodeB_AndStaysSticky() { _stubA.ThrowUnavailable = true; var reply = await CallAsync(); // Failed over: A was tried (and threw), B answered. Assert.Equal(LifecycleReply.ReplyOneofCase.InstanceLifecycle, reply.ReplyCase); Assert.Equal(1, _stubA.LifecycleCalls); Assert.Equal(1, _stubB.LifecycleCalls); Assert.False(_provider.IsOnPreferredNode(SiteId)); // Sticky: the next call goes straight to B without re-touching A. await CallAsync(); Assert.Equal(1, _stubA.LifecycleCalls); Assert.Equal(2, _stubB.LifecycleCalls); } [Fact] public async Task Failback_ReturnsToPreferredNodeA_OncePreferredIsReachableAgain() { _stubA.ThrowUnavailable = true; await CallAsync(); // flips to B Assert.False(_provider.IsOnPreferredNode(SiteId)); // NodeA recovers; the failback probe reports it reachable. _stubA.ThrowUnavailable = false; _preferredReachable = true; var back = await _provider.TryFailbackAsync(SiteId, CancellationToken.None); Assert.True(back); Assert.True(_provider.IsOnPreferredNode(SiteId)); var beforeA = _stubA.LifecycleCalls; await CallAsync(); Assert.Equal(beforeA + 1, _stubA.LifecycleCalls); // next call back on A } [Fact] public async Task DeadlineExceeded_OnNodeA_IsNotRetriedOnNodeB() { // A DeadlineExceeded is ambiguous (a WriteTag/Deploy/Failover may already have executed), so // — unlike Unavailable — it must NOT fail over to B. Modelled by A returning the status // directly, isolating the retry-decision from TestServer's own timeout mechanics. _stubA.StatusToThrow = StatusCode.DeadlineExceeded; var ex = await Assert.ThrowsAsync(() => CallAsync()); Assert.Equal(StatusCode.DeadlineExceeded, ex.StatusCode); Assert.Equal(1, _stubA.LifecycleCalls); Assert.Equal(0, _stubB.LifecycleCalls); // B was never tried Assert.True(_provider.IsOnPreferredNode(SiteId), "a deadline must not flip stickiness"); } [Fact] public async Task UnknownSite_Throws_SiteChannelUnavailable() { await Assert.ThrowsAsync( () => _provider.ExecuteAsync( "not-configured", (_, _) => Task.FromResult(new LifecycleReply()), CancellationToken.None)); } private sealed class FixedPskProvider(string key) : ISitePskProvider { public ValueTask GetAsync(string siteId, CancellationToken ct) => new(key); public void Invalidate(string siteId) { } } /// Stub SiteCommandService recording what the client sent and letting a test steer faults. private sealed class StubSiteCommandService : SiteCommandService.SiteCommandServiceBase { private int _lifecycleCalls; public int LifecycleCalls => Volatile.Read(ref _lifecycleCalls); public string? LastAuthHeader { get; private set; } public string? LastSiteHeader { get; private set; } public bool DeadlineWasSet { get; private set; } public volatile bool ThrowUnavailable; public StatusCode? StatusToThrow; public override Task ExecuteLifecycle(LifecycleRequest request, ServerCallContext context) { Interlocked.Increment(ref _lifecycleCalls); LastAuthHeader = context.RequestHeaders .FirstOrDefault(h => h.Key == ControlPlaneCredentials.AuthorizationHeader)?.Value; LastSiteHeader = context.RequestHeaders .FirstOrDefault(h => h.Key == ControlPlaneCredentials.SiteHeader)?.Value; DeadlineWasSet = context.Deadline != DateTime.MaxValue; if (ThrowUnavailable) { throw new RpcException(new Status(StatusCode.Unavailable, "node down")); } if (StatusToThrow is { } status) { throw new RpcException(new Status(status, "modelled fault")); } return Task.FromResult(SiteCommandDtoMapper.ToLifecycleReply( new InstanceLifecycleResponse("cmd-1", "Site1.Pump1", true, null, DateTimeOffset.UtcNow))); } } }