Joseph Doherty
213 lines
10 KiB
C#
213 lines
10 KiB
C#
using System.Net;
|
|
using System.Net.Http.Headers;
|
|
using Google.Protobuf;
|
|
using Grpc.Core;
|
|
using Grpc.Net.Client;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Microsoft.Extensions.Options;
|
|
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Grpc;
|
|
|
|
/// <summary>
|
|
/// Command-plane failback (review 2026-10-21 03/P-2): the probe must test READINESS with a real
|
|
/// RPC (a node that is Up but still warming refuses commands with a not-dispatched Unavailable),
|
|
/// and the backoff must only reset after a real call on the preferred node succeeds — not after
|
|
/// the probe — so a node that keeps failing real calls is retried ever less often.
|
|
/// </summary>
|
|
public class SitePairChannelProviderFailbackTests
|
|
{
|
|
private const string Site = "site-1";
|
|
|
|
private static SitePairChannelProvider Build(Func<GrpcChannel, CancellationToken, Task<bool>> probe) => new(
|
|
new NoKeyProvider(),
|
|
Options.Create(new CommunicationOptions()),
|
|
NullLogger<SitePairChannelProvider>.Instance,
|
|
handlerFactory: null,
|
|
reachabilityProbe: probe);
|
|
|
|
/// <summary>Runs one call that node A refuses as not-ready (then succeeds on B), as a warming node does.</summary>
|
|
private static async Task CallRejectedByA(SitePairChannelProvider provider)
|
|
{
|
|
GrpcChannel? first = null;
|
|
await provider.ExecuteAsync(Site, (channel, _) =>
|
|
{
|
|
if (first is null)
|
|
{
|
|
first = channel;
|
|
throw GrpcRetrySafety.NotDispatched("Site command plane not ready.");
|
|
}
|
|
return Task.FromResult(1);
|
|
}, CancellationToken.None);
|
|
}
|
|
|
|
private static Task CallSucceeds(SitePairChannelProvider provider) =>
|
|
provider.ExecuteAsync(Site, (_, _) => Task.FromResult(1), CancellationToken.None);
|
|
|
|
/// <summary>Verifies that a probe success alone does not reset the backoff; repeated rejections after failback grow it.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task ProbeSuccess_ThenRealCallRejected_GrowsTheBackoffInsteadOfResettingIt()
|
|
{
|
|
using var provider = Build((_, _) => Task.FromResult(true));
|
|
provider.UpdateSite(Site, "http://localhost:15301", "http://localhost:15302");
|
|
|
|
await CallRejectedByA(provider);
|
|
Assert.False(provider.IsOnPreferredNode(Site));
|
|
Assert.Equal(TimeSpan.FromSeconds(1), provider.GetFailbackBackoff(Site));
|
|
|
|
Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None));
|
|
await CallRejectedByA(provider);
|
|
Assert.Equal(TimeSpan.FromSeconds(2), provider.GetFailbackBackoff(Site));
|
|
|
|
Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None));
|
|
await CallRejectedByA(provider);
|
|
Assert.Equal(TimeSpan.FromSeconds(4), provider.GetFailbackBackoff(Site));
|
|
}
|
|
|
|
/// <summary>Verifies that a successful real call on the preferred node after failback resets the backoff.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task RealCallSucceedsAfterFailback_ResetsTheBackoff()
|
|
{
|
|
using var provider = Build((_, _) => Task.FromResult(true));
|
|
provider.UpdateSite(Site, "http://localhost:15311", "http://localhost:15312");
|
|
|
|
await CallRejectedByA(provider);
|
|
Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None));
|
|
await CallRejectedByA(provider);
|
|
Assert.Equal(TimeSpan.FromSeconds(2), provider.GetFailbackBackoff(Site));
|
|
|
|
Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None));
|
|
await CallSucceeds(provider); // A really is ready now
|
|
|
|
// A later, unrelated failure starts again from the initial backoff.
|
|
await CallRejectedByA(provider);
|
|
Assert.Equal(TimeSpan.FromSeconds(1), provider.GetFailbackBackoff(Site));
|
|
}
|
|
|
|
/// <summary>Verifies that probation expires: a quiet site whose NodeA dies long after the failback starts again at the initial backoff.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task ProbationOlderThanTheWindow_IsTreatedAsConfirmed_FirstProbeAtInitialBackoff()
|
|
{
|
|
using var provider = Build((_, _) => Task.FromResult(true));
|
|
provider.FailbackProbationWindow = TimeSpan.FromMilliseconds(100);
|
|
provider.UpdateSite(Site, "http://localhost:15321", "http://localhost:15322");
|
|
|
|
await CallRejectedByA(provider);
|
|
Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None));
|
|
await CallRejectedByA(provider);
|
|
Assert.Equal(TimeSpan.FromSeconds(2), provider.GetFailbackBackoff(Site));
|
|
|
|
// Fail back on probation, then the site goes quiet (no real call) past the window.
|
|
Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None));
|
|
await Task.Delay(300);
|
|
|
|
await CallRejectedByA(provider);
|
|
Assert.Equal(TimeSpan.FromSeconds(1), provider.GetFailbackBackoff(Site));
|
|
}
|
|
|
|
/// <summary>Verifies that an address change clears probation.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task AddressChange_ClearsProbation()
|
|
{
|
|
using var provider = Build((_, _) => Task.FromResult(true));
|
|
provider.UpdateSite(Site, "http://localhost:15331", "http://localhost:15332");
|
|
|
|
await CallRejectedByA(provider);
|
|
Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None));
|
|
await CallRejectedByA(provider);
|
|
Assert.True(await provider.TryFailbackAsync(Site, CancellationToken.None));
|
|
|
|
provider.UpdateSite(Site, "http://localhost:15333", "http://localhost:15332");
|
|
await CallRejectedByA(provider);
|
|
Assert.Equal(TimeSpan.FromSeconds(1), provider.GetFailbackBackoff(Site));
|
|
}
|
|
|
|
/// <summary>Verifies that the production probe treats a not-ready (not-dispatched Unavailable) site as not ready.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task ReadinessProbe_SiteRefusingAsNotReady_IsNotReady()
|
|
{
|
|
var handler = new GrpcStubHandler(ready: false);
|
|
using var channel = GrpcChannel.ForAddress("http://node-a", new GrpcChannelOptions { HttpHandler = handler });
|
|
|
|
Assert.False(await SitePairChannelProvider.ReadinessProbeAsync(channel, CancellationToken.None));
|
|
Assert.EndsWith(".SiteCommandService/ExecuteLifecycle", handler.LastPath);
|
|
}
|
|
|
|
/// <summary>Verifies that the production probe treats a site answering a real command as ready.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task ReadinessProbe_SiteAnsweringARealCommand_IsReady()
|
|
{
|
|
var handler = new GrpcStubHandler(ready: true);
|
|
using var channel = GrpcChannel.ForAddress("http://node-a", new GrpcChannelOptions { HttpHandler = handler });
|
|
|
|
Assert.True(await SitePairChannelProvider.ReadinessProbeAsync(channel, CancellationToken.None));
|
|
Assert.Contains(SitePairChannelProvider.ReadinessProbeInstance, handler.LastRequestBody);
|
|
}
|
|
|
|
/// <summary>Verifies that the probe's sentinel instance name can never be a real instance (it is not a legal actor name).</summary>
|
|
[Fact]
|
|
public void ReadinessProbeSentinel_IsNotALegalInstanceName()
|
|
{
|
|
Assert.False(ZB.MOM.WW.ScadaBridge.Commons.Validation.ActorNameRules.IsValid(
|
|
SitePairChannelProvider.ReadinessProbeInstance));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Minimal in-memory gRPC server stand-in: a trailers-only Unavailable (+ the not-dispatched
|
|
/// trailer) while not ready, or a framed <see cref="LifecycleReply"/> with OK status when ready.
|
|
/// </summary>
|
|
private sealed class GrpcStubHandler(bool ready) : HttpMessageHandler
|
|
{
|
|
public string LastPath { get; private set; } = "";
|
|
public string LastRequestBody { get; private set; } = "";
|
|
|
|
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken ct)
|
|
{
|
|
LastPath = request.RequestUri!.AbsolutePath;
|
|
var body = await request.Content!.ReadAsByteArrayAsync(ct);
|
|
var parsed = LifecycleRequest.Parser.ParseFrom(body.AsSpan(5).ToArray());
|
|
LastRequestBody = parsed.ToString();
|
|
|
|
var response = new HttpResponseMessage(HttpStatusCode.OK) { Version = HttpVersion.Version20 };
|
|
if (!ready)
|
|
{
|
|
var notDispatched = GrpcRetrySafety.NotDispatched("Site command plane not ready.");
|
|
response.Content = new ByteArrayContent(Array.Empty<byte>());
|
|
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/grpc");
|
|
response.Headers.Add("grpc-status", ((int)StatusCode.Unavailable).ToString());
|
|
foreach (var entry in notDispatched.Trailers)
|
|
{
|
|
response.Headers.Add(entry.Key, entry.Value);
|
|
}
|
|
return response;
|
|
}
|
|
|
|
var reply = new LifecycleReply { DeploymentStateQuery = new DeploymentStateQueryResponseDto() }.ToByteArray();
|
|
var framed = new byte[5 + reply.Length];
|
|
framed[1] = (byte)(reply.Length >> 24);
|
|
framed[2] = (byte)(reply.Length >> 16);
|
|
framed[3] = (byte)(reply.Length >> 8);
|
|
framed[4] = (byte)reply.Length;
|
|
reply.CopyTo(framed, 5);
|
|
response.Content = new ByteArrayContent(framed);
|
|
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/grpc");
|
|
response.TrailingHeaders.Add("grpc-status", "0");
|
|
return response;
|
|
}
|
|
}
|
|
|
|
private sealed class NoKeyProvider : ISitePskProvider
|
|
{
|
|
/// <inheritdoc />
|
|
public ValueTask<string> GetAsync(string siteId, CancellationToken ct) => new("k");
|
|
/// <inheritdoc />
|
|
public void Invalidate(string siteId) { }
|
|
}
|
|
}
|