using System.Net; using System.Net.Http.Headers; using Google.Protobuf; using Grpc.Core; using Grpc.Net.Client; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using ZB.MOM.WW.ScadaBridge.Communication.Grpc; namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Grpc; /// /// Command-plane failback (review 2026-10-21 03/P-2): the probe must test READINESS with a real /// RPC (a node that is Up but still warming refuses commands with a not-dispatched Unavailable), /// and the backoff must only reset after a real call on the preferred node succeeds — not after /// the probe — so a node that keeps failing real calls is retried ever less often. /// public class SitePairChannelProviderFailbackTests { private const string Site = "site-1"; private static SitePairChannelProvider Build(Func> probe) => new( new NoKeyProvider(), Options.Create(new CommunicationOptions()), NullLogger.Instance, handlerFactory: null, reachabilityProbe: probe); /// Runs one call that node A refuses as not-ready (then succeeds on B), as a warming node does. private static async Task CallRejectedByA(SitePairChannelProvider provider) { GrpcChannel? first = null; await provider.ExecuteAsync(Site, (channel, _) => { if (first is null) { first = channel; throw GrpcRetrySafety.NotDispatched("Site command plane not ready."); } return Task.FromResult(1); }, CancellationToken.None); } private static Task CallSucceeds(SitePairChannelProvider provider) => provider.ExecuteAsync(Site, (_, _) => Task.FromResult(1), CancellationToken.None); /// Verifies that a probe success alone does not reset the backoff; repeated rejections after failback grow it. /// A task that represents the asynchronous operation. [Fact] public async Task ProbeSuccess_ThenRealCallRejected_GrowsTheBackoffInsteadOfResettingIt() { using var provider = Build((_, _) => Task.FromResult(true)); provider.UpdateSite(Site, "http://localhost:15301", "http://localhost:15302"); await CallRejectedByA(provider); Assert.False(provider.IsOnPreferredNode(Site)); Assert.Equal(TimeSpan.FromSeconds(1), provider.GetFailbackBackoff(Site)); Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None)); await CallRejectedByA(provider); Assert.Equal(TimeSpan.FromSeconds(2), provider.GetFailbackBackoff(Site)); Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None)); await CallRejectedByA(provider); Assert.Equal(TimeSpan.FromSeconds(4), provider.GetFailbackBackoff(Site)); } /// Verifies that a successful real call on the preferred node after failback resets the backoff. /// A task that represents the asynchronous operation. [Fact] public async Task RealCallSucceedsAfterFailback_ResetsTheBackoff() { using var provider = Build((_, _) => Task.FromResult(true)); provider.UpdateSite(Site, "http://localhost:15311", "http://localhost:15312"); await CallRejectedByA(provider); Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None)); await CallRejectedByA(provider); Assert.Equal(TimeSpan.FromSeconds(2), provider.GetFailbackBackoff(Site)); Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None)); await CallSucceeds(provider); // A really is ready now // A later, unrelated failure starts again from the initial backoff. await CallRejectedByA(provider); Assert.Equal(TimeSpan.FromSeconds(1), provider.GetFailbackBackoff(Site)); } /// Verifies that probation expires: a quiet site whose NodeA dies long after the failback starts again at the initial backoff. /// A task that represents the asynchronous operation. [Fact] public async Task ProbationOlderThanTheWindow_IsTreatedAsConfirmed_FirstProbeAtInitialBackoff() { using var provider = Build((_, _) => Task.FromResult(true)); provider.FailbackProbationWindow = TimeSpan.FromMilliseconds(100); provider.UpdateSite(Site, "http://localhost:15321", "http://localhost:15322"); await CallRejectedByA(provider); Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None)); await CallRejectedByA(provider); Assert.Equal(TimeSpan.FromSeconds(2), provider.GetFailbackBackoff(Site)); // Fail back on probation, then the site goes quiet (no real call) past the window. Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None)); await Task.Delay(300); await CallRejectedByA(provider); Assert.Equal(TimeSpan.FromSeconds(1), provider.GetFailbackBackoff(Site)); } /// Verifies that an address change clears probation. /// A task that represents the asynchronous operation. [Fact] public async Task AddressChange_ClearsProbation() { using var provider = Build((_, _) => Task.FromResult(true)); provider.UpdateSite(Site, "http://localhost:15331", "http://localhost:15332"); await CallRejectedByA(provider); Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None)); await CallRejectedByA(provider); Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None)); provider.UpdateSite(Site, "http://localhost:15333", "http://localhost:15332"); await CallRejectedByA(provider); Assert.Equal(TimeSpan.FromSeconds(1), provider.GetFailbackBackoff(Site)); } /// Verifies that the production probe treats a not-ready (not-dispatched Unavailable) site as not ready. /// A task that represents the asynchronous operation. [Fact] public async Task ReadinessProbe_SiteRefusingAsNotReady_IsNotReady() { var handler = new GrpcStubHandler(ready: false); using var channel = GrpcChannel.ForAddress("http://node-a", new GrpcChannelOptions { HttpHandler = handler }); Assert.False(await SitePairChannelProvider.ReadinessProbeAsync(channel, CancellationToken.None)); Assert.EndsWith(".SiteCommandService/ExecuteLifecycle", handler.LastPath); } /// Verifies that the production probe treats a site answering a real command as ready. /// A task that represents the asynchronous operation. [Fact] public async Task ReadinessProbe_SiteAnsweringARealCommand_IsReady() { var handler = new GrpcStubHandler(ready: true); using var channel = GrpcChannel.ForAddress("http://node-a", new GrpcChannelOptions { HttpHandler = handler }); Assert.True(await SitePairChannelProvider.ReadinessProbeAsync(channel, CancellationToken.None)); Assert.Contains(SitePairChannelProvider.ReadinessProbeInstance, handler.LastRequestBody); } /// Verifies that the probe's sentinel instance name can never be a real instance (it is not a legal actor name). [Fact] public void ReadinessProbeSentinel_IsNotALegalInstanceName() { Assert.False(ZB.MOM.WW.ScadaBridge.Commons.Validation.ActorNameRules.IsValid( SitePairChannelProvider.ReadinessProbeInstance)); } /// /// Minimal in-memory gRPC server stand-in: a trailers-only Unavailable (+ the not-dispatched /// trailer) while not ready, or a framed with OK status when ready. /// private sealed class GrpcStubHandler(bool ready) : HttpMessageHandler { public string LastPath { get; private set; } = ""; public string LastRequestBody { get; private set; } = ""; protected override async Task SendAsync(HttpRequestMessage request, CancellationToken ct) { LastPath = request.RequestUri!.AbsolutePath; var body = await request.Content!.ReadAsByteArrayAsync(ct); var parsed = LifecycleRequest.Parser.ParseFrom(body.AsSpan(5).ToArray()); LastRequestBody = parsed.ToString(); var response = new HttpResponseMessage(HttpStatusCode.OK) { Version = HttpVersion.Version20 }; if (!ready) { var notDispatched = GrpcRetrySafety.NotDispatched("Site command plane not ready."); response.Content = new ByteArrayContent(Array.Empty()); response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/grpc"); response.Headers.Add("grpc-status", ((int)StatusCode.Unavailable).ToString()); foreach (var entry in notDispatched.Trailers) { response.Headers.Add(entry.Key, entry.Value); } return response; } var reply = new LifecycleReply { DeploymentStateQuery = new DeploymentStateQueryResponseDto() }.ToByteArray(); var framed = new byte[5 + reply.Length]; framed[1] = (byte)(reply.Length >> 24); framed[2] = (byte)(reply.Length >> 16); framed[3] = (byte)(reply.Length >> 8); framed[4] = (byte)reply.Length; reply.CopyTo(framed, 5); response.Content = new ByteArrayContent(framed); response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/grpc"); response.TrailingHeaders.Add("grpc-status", "0"); return response; } } private sealed class NoKeyProvider : ISitePskProvider { /// public ValueTask GetAsync(string siteId, CancellationToken ct) => new("k"); /// public void Invalidate(string siteId) { } } }