fix(communication): command-plane failback probes readiness with a real RPC; backoff resets only after a real call succeeds (review 2026-10-21 03/P-2)
This commit is contained in:
Joseph Doherty
+164
@@ -0,0 +1,164 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using Google.Protobuf;
|
||||
using Grpc.Core;
|
||||
using Grpc.Net.Client;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// Command-plane failback (review 2026-10-21 03/P-2): the probe must test READINESS with a real
|
||||
/// RPC (a node that is Up but still warming refuses commands with a not-dispatched Unavailable),
|
||||
/// and the backoff must only reset after a real call on the preferred node succeeds — not after
|
||||
/// the probe — so a node that keeps failing real calls is retried ever less often.
|
||||
/// </summary>
|
||||
public class SitePairChannelProviderFailbackTests
|
||||
{
|
||||
private const string Site = "site-1";
|
||||
|
||||
private static SitePairChannelProvider Build(Func<GrpcChannel, CancellationToken, Task<bool>> probe) => new(
|
||||
new NoKeyProvider(),
|
||||
Options.Create(new CommunicationOptions()),
|
||||
NullLogger<SitePairChannelProvider>.Instance,
|
||||
handlerFactory: null,
|
||||
reachabilityProbe: probe);
|
||||
|
||||
/// <summary>Runs one call that node A refuses as not-ready (then succeeds on B), as a warming node does.</summary>
|
||||
private static async Task CallRejectedByA(SitePairChannelProvider provider)
|
||||
{
|
||||
GrpcChannel? first = null;
|
||||
await provider.ExecuteAsync(Site, (channel, _) =>
|
||||
{
|
||||
if (first is null)
|
||||
{
|
||||
first = channel;
|
||||
throw GrpcRetrySafety.NotDispatched("Site command plane not ready.");
|
||||
}
|
||||
return Task.FromResult(1);
|
||||
}, CancellationToken.None);
|
||||
}
|
||||
|
||||
private static Task CallSucceeds(SitePairChannelProvider provider) =>
|
||||
provider.ExecuteAsync(Site, (_, _) => Task.FromResult(1), CancellationToken.None);
|
||||
|
||||
/// <summary>Verifies that a probe success alone does not reset the backoff; repeated rejections after failback grow it.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ProbeSuccess_ThenRealCallRejected_GrowsTheBackoffInsteadOfResettingIt()
|
||||
{
|
||||
using var provider = Build((_, _) => Task.FromResult(true));
|
||||
provider.UpdateSite(Site, "http://localhost:15301", "http://localhost:15302");
|
||||
|
||||
await CallRejectedByA(provider);
|
||||
Assert.False(provider.IsOnPreferredNode(Site));
|
||||
Assert.Equal(TimeSpan.FromSeconds(1), provider.GetFailbackBackoff(Site));
|
||||
|
||||
Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None));
|
||||
await CallRejectedByA(provider);
|
||||
Assert.Equal(TimeSpan.FromSeconds(2), provider.GetFailbackBackoff(Site));
|
||||
|
||||
Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None));
|
||||
await CallRejectedByA(provider);
|
||||
Assert.Equal(TimeSpan.FromSeconds(4), provider.GetFailbackBackoff(Site));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a successful real call on the preferred node after failback resets the backoff.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RealCallSucceedsAfterFailback_ResetsTheBackoff()
|
||||
{
|
||||
using var provider = Build((_, _) => Task.FromResult(true));
|
||||
provider.UpdateSite(Site, "http://localhost:15311", "http://localhost:15312");
|
||||
|
||||
await CallRejectedByA(provider);
|
||||
Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None));
|
||||
await CallRejectedByA(provider);
|
||||
Assert.Equal(TimeSpan.FromSeconds(2), provider.GetFailbackBackoff(Site));
|
||||
|
||||
Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None));
|
||||
await CallSucceeds(provider); // A really is ready now
|
||||
|
||||
// A later, unrelated failure starts again from the initial backoff.
|
||||
await CallRejectedByA(provider);
|
||||
Assert.Equal(TimeSpan.FromSeconds(1), provider.GetFailbackBackoff(Site));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the production probe treats a not-ready (not-dispatched Unavailable) site as not ready.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ReadinessProbe_SiteRefusingAsNotReady_IsNotReady()
|
||||
{
|
||||
var handler = new GrpcStubHandler(ready: false);
|
||||
using var channel = GrpcChannel.ForAddress("http://node-a", new GrpcChannelOptions { HttpHandler = handler });
|
||||
|
||||
Assert.False(await SitePairChannelProvider.ReadinessProbeAsync(channel, CancellationToken.None));
|
||||
Assert.EndsWith(".SiteCommandService/ExecuteLifecycle", handler.LastPath);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the production probe treats a site answering a real command as ready.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ReadinessProbe_SiteAnsweringARealCommand_IsReady()
|
||||
{
|
||||
var handler = new GrpcStubHandler(ready: true);
|
||||
using var channel = GrpcChannel.ForAddress("http://node-a", new GrpcChannelOptions { HttpHandler = handler });
|
||||
|
||||
Assert.True(await SitePairChannelProvider.ReadinessProbeAsync(channel, CancellationToken.None));
|
||||
Assert.Contains(SitePairChannelProvider.ReadinessProbeInstance, handler.LastRequestBody);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Minimal in-memory gRPC server stand-in: a trailers-only Unavailable (+ the not-dispatched
|
||||
/// trailer) while not ready, or a framed <see cref="LifecycleReply"/> with OK status when ready.
|
||||
/// </summary>
|
||||
private sealed class GrpcStubHandler(bool ready) : HttpMessageHandler
|
||||
{
|
||||
public string LastPath { get; private set; } = "";
|
||||
public string LastRequestBody { get; private set; } = "";
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken ct)
|
||||
{
|
||||
LastPath = request.RequestUri!.AbsolutePath;
|
||||
var body = await request.Content!.ReadAsByteArrayAsync(ct);
|
||||
var parsed = LifecycleRequest.Parser.ParseFrom(body.AsSpan(5).ToArray());
|
||||
LastRequestBody = parsed.ToString();
|
||||
|
||||
var response = new HttpResponseMessage(HttpStatusCode.OK) { Version = HttpVersion.Version20 };
|
||||
if (!ready)
|
||||
{
|
||||
var notDispatched = GrpcRetrySafety.NotDispatched("Site command plane not ready.");
|
||||
response.Content = new ByteArrayContent(Array.Empty<byte>());
|
||||
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/grpc");
|
||||
response.Headers.Add("grpc-status", ((int)StatusCode.Unavailable).ToString());
|
||||
foreach (var entry in notDispatched.Trailers)
|
||||
{
|
||||
response.Headers.Add(entry.Key, entry.Value);
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
var reply = new LifecycleReply { DeploymentStateQuery = new DeploymentStateQueryResponseDto() }.ToByteArray();
|
||||
var framed = new byte[5 + reply.Length];
|
||||
framed[1] = (byte)(reply.Length >> 24);
|
||||
framed[2] = (byte)(reply.Length >> 16);
|
||||
framed[3] = (byte)(reply.Length >> 8);
|
||||
framed[4] = (byte)reply.Length;
|
||||
reply.CopyTo(framed, 5);
|
||||
response.Content = new ByteArrayContent(framed);
|
||||
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/grpc");
|
||||
response.TrailingHeaders.Add("grpc-status", "0");
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NoKeyProvider : ISitePskProvider
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public ValueTask<string> GetAsync(string siteId, CancellationToken ct) => new("k");
|
||||
/// <inheritdoc />
|
||||
public void Invalidate(string siteId) { }
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user