feat(comm): T1B.3 — central-side gRPC site-command transport seam (default Akka)
Extract the central→site send path in CentralCommunicationActor behind a new ISiteCommandTransport, selected by ScadaBridge:Communication:SiteTransport (Akka | Grpc, default Akka — rollback = flip the flag). CommunicationService's 27 commands, SiteCallAuditActor's 2 parked relays and DebugStreamBridgeActor's subscribe/unsubscribe are untouched; the seam sits below SiteEnvelope. - AkkaSiteTransport: today's per-site ClusterClient path extracted verbatim (the _siteClients lookup + ClusterClient.Send with the reply-to sender preserved, and the "no client ⇒ warn + drop, caller's Ask times out" path). - GrpcSiteTransport: dials the site SiteCommandService (T1B.1 proto client) via SiteCommandDtoMapper, PSK + x-scadabridge-site on the channel through ControlPlaneCredentials, per-command deadlines set EQUAL to today's CommunicationService Ask timeouts (per-command, not per-group: DeploymentState query and TriggerSiteFailover use QueryTimeout; the two parked relays map to QueryTimeout so SiteCallAudit's inner RelayTimeout 10s < 30s ordering holds; WaitForAttribute keeps its dynamic Timeout + IntegrationTimeout). - SitePairChannelProvider: per-site A/B channel pair with sticky failover (flip only on Unavailable — NEVER on DeadlineExceeded, a write/deploy/failover may have run), background failback probe to the preferred node with 1s→60s doubling backoff, PSK invalidation on site removal. Fed by the SAME DB refresh loop (extended to carry GrpcNodeA/GrpcNodeBAddress) — no second poll. Tests: actor-with-substitute-transport (routing, Ask-reply plumbing, per-site lifecycle across refreshes), ResolveDeadline pinned to each command's current Ask timeout, and GrpcSiteTransport/SitePairChannelProvider over dual in-process TestServers (PSK+header+deadline attached, Unavailable failover + stickiness, failback to preferred, no-retry-on-DeadlineExceeded). Proto csproj untouched (no active <Protobuf> item). Full solution builds 0 warnings; Communication.Tests 607 green with Akka default.
This commit is contained in:
Joseph Doherty
+2
-1
@@ -39,7 +39,8 @@ public class CentralCommunicationActorClientLifecycleTests : TestKit
|
||||
private static SiteAddressCacheLoaded Load(string siteId, params string[] addrs) =>
|
||||
new(new Dictionary<string, IReadOnlyList<string>>
|
||||
{ [siteId] = addrs.ToList().AsReadOnly() },
|
||||
new[] { siteId });
|
||||
new[] { siteId },
|
||||
new Dictionary<string, SiteGrpcEndpoints>());
|
||||
|
||||
[Fact]
|
||||
public void PeriodicRefresh_PrunesDeletedSites_FromHealthAggregator()
|
||||
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// T1B.3 seam tests: <see cref="CentralCommunicationActor"/> routing a <see cref="SiteEnvelope"/>
|
||||
/// through an injected <see cref="ISiteCommandTransport"/> (transport-agnostic), proving the
|
||||
/// envelope reaches the transport, the reply routes back to the waiting Ask, and the DB refresh
|
||||
/// tick reconciles per-site transport resources.
|
||||
/// </summary>
|
||||
public class CentralCommunicationActorTransportTests : TestKit
|
||||
{
|
||||
public CentralCommunicationActorTransportTests() : base(@"akka.loglevel = WARNING") { }
|
||||
|
||||
private static Site GrpcSite(string id, string? grpcA = "http://a:8083", string? grpcB = "http://b:8083") =>
|
||||
new("Test " + id, id)
|
||||
{
|
||||
NodeAAddress = $"akka.tcp://scadabridge@{id}-a:8081",
|
||||
GrpcNodeAAddress = grpcA,
|
||||
GrpcNodeBAddress = grpcB
|
||||
};
|
||||
|
||||
private (IActorRef actor, ISiteCommandTransport transport, ISiteRepository repo) CreateActor(
|
||||
IEnumerable<Site>? sites = null)
|
||||
{
|
||||
var repo = Substitute.For<ISiteRepository>();
|
||||
repo.GetAllSitesAsync(Arg.Any<CancellationToken>()).Returns(sites?.ToList() ?? new List<Site>());
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddScoped(_ => repo);
|
||||
var sp = services.BuildServiceProvider();
|
||||
|
||||
var transport = Substitute.For<ISiteCommandTransport>();
|
||||
var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, transport, (TimeSpan?)null)));
|
||||
return (actor, transport, repo);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SiteEnvelope_IsRoutedToTheTransport_WithTheSenderAsReplyTo()
|
||||
{
|
||||
var (actor, transport, _) = CreateActor();
|
||||
var probe = CreateTestProbe();
|
||||
|
||||
var cmd = new EnableInstanceCommand("cmd-1", "Site1.Pump1", DateTimeOffset.UtcNow);
|
||||
actor.Tell(new SiteEnvelope("site-a", cmd), probe.Ref);
|
||||
|
||||
AwaitAssert(() => transport.Received(1).Send(
|
||||
Arg.Is<SiteEnvelope>(e => e.SiteId == "site-a" && ReferenceEquals(e.Message, cmd)),
|
||||
probe.Ref));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AskReply_FromTransport_RoutesBackToTheWaitingAsk()
|
||||
{
|
||||
var (actor, transport, _) = CreateActor();
|
||||
|
||||
// The transport substitute stands in for the site: when handed the envelope it delivers a
|
||||
// reply to the captured replyTo, exactly as a real transport pipes the site's reply back.
|
||||
var reply = new DeploymentStatusResponse(
|
||||
"dep-1", "Site1.Pump1", DeploymentStatus.Success, null, DateTimeOffset.UtcNow);
|
||||
transport
|
||||
.When(t => t.Send(Arg.Any<SiteEnvelope>(), Arg.Any<IActorRef>()))
|
||||
.Do(ci => ci.Arg<IActorRef>().Tell(reply));
|
||||
|
||||
var cmd = new RefreshDeploymentCommand(
|
||||
"dep-1", "Site1.Pump1", "hash", "multi-role", DateTimeOffset.UtcNow, "http://c:5000", "tok");
|
||||
var result = await actor.Ask<DeploymentStatusResponse>(
|
||||
new SiteEnvelope("site-a", cmd), TimeSpan.FromSeconds(3));
|
||||
|
||||
Assert.Equal("dep-1", result.DeploymentId);
|
||||
Assert.Equal(DeploymentStatus.Success, result.Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DbRefresh_ReconcilesTheTransport_WithTheLoadedGrpcEndpoints()
|
||||
{
|
||||
var (_, transport, _) = CreateActor(new[] { GrpcSite("site-a") });
|
||||
|
||||
// PreStart fires the refresh at Zero; the loaded cache is handed to the transport.
|
||||
AwaitAssert(() => transport.Received().ReconcileSites(
|
||||
Arg.Is<SiteAddressCacheLoaded>(c =>
|
||||
c.KnownSiteIds.Contains("site-a")
|
||||
&& c.GrpcContacts.ContainsKey("site-a")
|
||||
&& c.GrpcContacts["site-a"].NodeA == "http://a:8083"
|
||||
&& c.GrpcContacts["site-a"].NodeB == "http://b:8083")),
|
||||
TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SiteAddedAndRemoved_AcrossRefreshes_FlowsThroughToTheTransport()
|
||||
{
|
||||
var (actor, transport, repo) = CreateActor(new[] { GrpcSite("site-a") });
|
||||
|
||||
AwaitAssert(() => transport.Received().ReconcileSites(
|
||||
Arg.Is<SiteAddressCacheLoaded>(c => c.GrpcContacts.ContainsKey("site-a"))),
|
||||
TimeSpan.FromSeconds(3));
|
||||
|
||||
// site-a removed, site-b added.
|
||||
repo.GetAllSitesAsync(Arg.Any<CancellationToken>()).Returns(new List<Site> { GrpcSite("site-b") });
|
||||
actor.Tell(new RefreshSiteAddresses());
|
||||
|
||||
AwaitAssert(() => transport.Received().ReconcileSites(
|
||||
Arg.Is<SiteAddressCacheLoaded>(c =>
|
||||
c.GrpcContacts.ContainsKey("site-b")
|
||||
&& !c.GrpcContacts.ContainsKey("site-a")
|
||||
&& c.KnownSiteIds.Contains("site-b")
|
||||
&& !c.KnownSiteIds.Contains("site-a"))),
|
||||
TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SiteWithoutGrpcAddresses_IsAbsentFromGrpcContacts_ButStillKnown()
|
||||
{
|
||||
// A site with only Akka addresses (no gRPC columns) is a known site but carries no gRPC
|
||||
// endpoint — the gRPC transport must not try to dial it, the Akka one still can.
|
||||
var akkaOnly = new Site("Akka only", "site-x")
|
||||
{
|
||||
NodeAAddress = "akka.tcp://scadabridge@site-x-a:8081"
|
||||
};
|
||||
var (_, transport, _) = CreateActor(new[] { akkaOnly });
|
||||
|
||||
AwaitAssert(() => transport.Received().ReconcileSites(
|
||||
Arg.Is<SiteAddressCacheLoaded>(c =>
|
||||
c.KnownSiteIds.Contains("site-x") && !c.GrpcContacts.ContainsKey("site-x"))),
|
||||
TimeSpan.FromSeconds(3));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.InboundApi;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Management;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.RemoteQuery;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Pins <see cref="GrpcSiteTransport.ResolveDeadline"/> to the EXACT Ask timeout
|
||||
/// <see cref="CommunicationService"/> uses for each command today, so flipping the transport cannot
|
||||
/// change how long any call waits. Every timeout is given a distinct value so a wrong mapping is
|
||||
/// caught, not masked by two defaults that happen to be equal (QueryTimeout == IntegrationTimeout in
|
||||
/// production).
|
||||
/// </summary>
|
||||
public class GrpcSiteTransportDeadlineTests
|
||||
{
|
||||
private static readonly CommunicationOptions Opts = new()
|
||||
{
|
||||
DeploymentTimeout = TimeSpan.FromSeconds(120),
|
||||
LifecycleTimeout = TimeSpan.FromSeconds(30),
|
||||
ArtifactDeploymentTimeout = TimeSpan.FromSeconds(60),
|
||||
QueryTimeout = TimeSpan.FromSeconds(31),
|
||||
IntegrationTimeout = TimeSpan.FromSeconds(32),
|
||||
DebugViewTimeout = TimeSpan.FromSeconds(10)
|
||||
};
|
||||
|
||||
private static readonly GrpcSiteTransport Transport = new(
|
||||
new SitePairChannelProvider(
|
||||
new NoKeyProvider(), Options.Create(Opts), NullLogger<SitePairChannelProvider>.Instance),
|
||||
Opts,
|
||||
NullLogger<GrpcSiteTransport>.Instance);
|
||||
|
||||
private static readonly DateTimeOffset T = DateTimeOffset.UtcNow;
|
||||
|
||||
public static IEnumerable<object[]> Cases()
|
||||
{
|
||||
// Lifecycle group — note it is NOT one deadline class.
|
||||
yield return [new RefreshDeploymentCommand("d", "i", "h", "by", T, "u", "t"), Opts.DeploymentTimeout];
|
||||
yield return [new EnableInstanceCommand("c", "i", T), Opts.LifecycleTimeout];
|
||||
yield return [new DisableInstanceCommand("c", "i", T), Opts.LifecycleTimeout];
|
||||
yield return [new DeleteInstanceCommand("c", "i", T), Opts.LifecycleTimeout];
|
||||
// DeploymentStateQuery uses QueryTimeout in CommunicationService, NOT LifecycleTimeout.
|
||||
yield return [new DeploymentStateQueryRequest("c", "i", T), Opts.QueryTimeout];
|
||||
yield return [new DeployArtifactsCommand("d", null, null, null, null, null, null, T), Opts.ArtifactDeploymentTimeout];
|
||||
|
||||
// OPC UA group — all QueryTimeout.
|
||||
yield return [new BrowseNodeCommand("conn", null, null, null), Opts.QueryTimeout];
|
||||
yield return [new SearchAddressSpaceCommand("conn", "q", 1, 1, null), Opts.QueryTimeout];
|
||||
yield return [new ReadTagValuesCommand("conn", []), Opts.QueryTimeout];
|
||||
yield return [new VerifyEndpointCommand("conn", "OpcUa", "{}", null), Opts.QueryTimeout];
|
||||
yield return [new TrustServerCertCommand("conn", "ZGVy", "AA", null), Opts.QueryTimeout];
|
||||
yield return [new ListServerCertsCommand(null), Opts.QueryTimeout];
|
||||
yield return [new RemoveServerCertCommand("AA", null), Opts.QueryTimeout];
|
||||
yield return [new WriteTagRequest("c", "conn", "tag", 1, T), Opts.QueryTimeout];
|
||||
|
||||
// Query group.
|
||||
yield return [new EventLogQueryRequest("c", "s", null, null, null, null, null, null, null, 10, T), Opts.QueryTimeout];
|
||||
yield return [new DebugSnapshotRequest("i", "c"), Opts.QueryTimeout];
|
||||
yield return [new SubscribeDebugViewRequest("i", "c"), Opts.DebugViewTimeout];
|
||||
yield return [new UnsubscribeDebugViewRequest("i", "c"), Opts.DebugViewTimeout];
|
||||
|
||||
// Parked group — the two relays keep QueryTimeout (30s) so SiteCallAudit's inner
|
||||
// RelayTimeout (10s) still expires first.
|
||||
yield return [new ParkedMessageQueryRequest("c", "s", 1, 10, T), Opts.QueryTimeout];
|
||||
yield return [new ParkedMessageRetryRequest("c", "s", "m", T), Opts.QueryTimeout];
|
||||
yield return [new ParkedMessageDiscardRequest("c", "s", "m", T), Opts.QueryTimeout];
|
||||
yield return [new RetryParkedOperation("c", new TrackedOperationId(Guid.NewGuid())), Opts.QueryTimeout];
|
||||
yield return [new DiscardParkedOperation("c", new TrackedOperationId(Guid.NewGuid())), Opts.QueryTimeout];
|
||||
|
||||
// Route group.
|
||||
yield return [new RouteToCallRequest("c", "i", "m", null, T), Opts.IntegrationTimeout];
|
||||
yield return [new RouteToGetAttributesRequest("c", "i", [], T), Opts.IntegrationTimeout];
|
||||
yield return [new RouteToSetAttributesRequest("c", "i", new Dictionary<string, string>(), T), Opts.IntegrationTimeout];
|
||||
|
||||
// Failover — QueryTimeout in CommunicationService, NOT LifecycleTimeout.
|
||||
yield return [new TriggerSiteFailover("c", "s"), Opts.QueryTimeout];
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(Cases))]
|
||||
public void ResolveDeadline_MatchesTodaysAskTimeout(object command, TimeSpan expected)
|
||||
{
|
||||
Assert.Equal(expected, Transport.ResolveDeadline(command));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WaitForAttribute_UsesItsDynamicTimeoutPlusIntegrationSlack()
|
||||
{
|
||||
// CommunicationService.RouteToWaitForAttributeAsync uses request.Timeout + IntegrationTimeout.
|
||||
var wait = new RouteToWaitForAttributeRequest("c", "i", "attr", "10", TimeSpan.FromSeconds(45), T);
|
||||
Assert.Equal(TimeSpan.FromSeconds(45) + Opts.IntegrationTimeout, Transport.ResolveDeadline(wait));
|
||||
}
|
||||
|
||||
private sealed class NoKeyProvider : ISitePskProvider
|
||||
{
|
||||
public ValueTask<string> GetAsync(string siteId, CancellationToken ct) => new("k");
|
||||
public void Invalidate(string siteId) { }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
using Grpc.Core;
|
||||
using Grpc.Net.Client;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.TestHost;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// T1B.3: <see cref="SitePairChannelProvider"/> failover/failback + credential/deadline proof over
|
||||
/// two in-process gRPC <see cref="TestServer"/>s (NodeA preferred, NodeB standby). Exercises the
|
||||
/// real client stack — PSK call credentials, per-call deadline, sticky failover, no-retry on
|
||||
/// <see cref="StatusCode.DeadlineExceeded"/>, and failback to the preferred node.
|
||||
/// </summary>
|
||||
public sealed class GrpcSiteTransportFailoverTests : IAsyncLifetime
|
||||
{
|
||||
private const string SiteId = "site-1";
|
||||
private const string SiteKey = "the-site-1-key";
|
||||
private const string EndpointA = "http://node-a";
|
||||
private const string EndpointB = "http://node-b";
|
||||
|
||||
private IHost _hostA = null!;
|
||||
private IHost _hostB = null!;
|
||||
private StubSiteCommandService _stubA = null!;
|
||||
private StubSiteCommandService _stubB = null!;
|
||||
private SitePairChannelProvider _provider = null!;
|
||||
private volatile bool _preferredReachable;
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
(_hostA, _stubA) = await StartServerAsync();
|
||||
(_hostB, _stubB) = await StartServerAsync();
|
||||
|
||||
var handlerA = _hostA.GetTestServer().CreateHandler();
|
||||
var handlerB = _hostB.GetTestServer().CreateHandler();
|
||||
|
||||
_provider = new SitePairChannelProvider(
|
||||
new FixedPskProvider(SiteKey),
|
||||
Options.Create(new CommunicationOptions()),
|
||||
NullLogger<SitePairChannelProvider>.Instance,
|
||||
handlerFactory: endpoint => endpoint == EndpointA ? handlerA : handlerB,
|
||||
reachabilityProbe: (_, _) => Task.FromResult(_preferredReachable));
|
||||
|
||||
_provider.UpdateSite(SiteId, EndpointA, EndpointB);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
_provider.Dispose();
|
||||
await _hostA.StopAsync();
|
||||
await _hostB.StopAsync();
|
||||
_hostA.Dispose();
|
||||
_hostB.Dispose();
|
||||
}
|
||||
|
||||
private static async Task<(IHost, StubSiteCommandService)> StartServerAsync()
|
||||
{
|
||||
var stub = new StubSiteCommandService();
|
||||
var host = await new HostBuilder()
|
||||
.ConfigureWebHost(web => web
|
||||
.UseTestServer()
|
||||
.ConfigureServices(services =>
|
||||
{
|
||||
services.AddGrpc();
|
||||
services.AddSingleton(stub);
|
||||
})
|
||||
.Configure(app =>
|
||||
{
|
||||
app.UseRouting();
|
||||
app.UseEndpoints(e => e.MapGrpcService<StubSiteCommandService>());
|
||||
}))
|
||||
.StartAsync();
|
||||
return (host, stub);
|
||||
}
|
||||
|
||||
private static LifecycleRequest EnableReq() =>
|
||||
SiteCommandDtoMapper.ToLifecycleRequest(
|
||||
new EnableInstanceCommand("cmd-1", "Site1.Pump1", DateTimeOffset.UtcNow));
|
||||
|
||||
private Task<LifecycleReply> CallAsync(TimeSpan? deadline = null) =>
|
||||
_provider.ExecuteAsync(
|
||||
SiteId,
|
||||
(channel, ct) =>
|
||||
{
|
||||
var client = new SiteCommandService.SiteCommandServiceClient(channel);
|
||||
return client.ExecuteLifecycleAsync(
|
||||
EnableReq(),
|
||||
deadline: DateTime.UtcNow + (deadline ?? TimeSpan.FromSeconds(10)),
|
||||
cancellationToken: ct).ResponseAsync;
|
||||
},
|
||||
CancellationToken.None);
|
||||
|
||||
[Fact]
|
||||
public async Task HealthyCall_HitsPreferredNodeA_WithPskAndSiteHeaderAndDeadline()
|
||||
{
|
||||
var reply = await CallAsync();
|
||||
|
||||
Assert.Equal(LifecycleReply.ReplyOneofCase.InstanceLifecycle, reply.ReplyCase);
|
||||
Assert.Equal(1, _stubA.LifecycleCalls);
|
||||
Assert.Equal(0, _stubB.LifecycleCalls);
|
||||
Assert.Equal($"Bearer {SiteKey}", _stubA.LastAuthHeader);
|
||||
Assert.Equal(SiteId, _stubA.LastSiteHeader);
|
||||
Assert.True(_stubA.DeadlineWasSet, "the client must set a per-call deadline");
|
||||
Assert.True(_provider.IsOnPreferredNode(SiteId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Unavailable_OnNodeA_FailsOverToNodeB_AndStaysSticky()
|
||||
{
|
||||
_stubA.ThrowUnavailable = true;
|
||||
|
||||
var reply = await CallAsync();
|
||||
|
||||
// Failed over: A was tried (and threw), B answered.
|
||||
Assert.Equal(LifecycleReply.ReplyOneofCase.InstanceLifecycle, reply.ReplyCase);
|
||||
Assert.Equal(1, _stubA.LifecycleCalls);
|
||||
Assert.Equal(1, _stubB.LifecycleCalls);
|
||||
Assert.False(_provider.IsOnPreferredNode(SiteId));
|
||||
|
||||
// Sticky: the next call goes straight to B without re-touching A.
|
||||
await CallAsync();
|
||||
Assert.Equal(1, _stubA.LifecycleCalls);
|
||||
Assert.Equal(2, _stubB.LifecycleCalls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Failback_ReturnsToPreferredNodeA_OncePreferredIsReachableAgain()
|
||||
{
|
||||
_stubA.ThrowUnavailable = true;
|
||||
await CallAsync(); // flips to B
|
||||
Assert.False(_provider.IsOnPreferredNode(SiteId));
|
||||
|
||||
// NodeA recovers; the failback probe reports it reachable.
|
||||
_stubA.ThrowUnavailable = false;
|
||||
_preferredReachable = true;
|
||||
var back = await _provider.TryFailbackAsync(SiteId, CancellationToken.None);
|
||||
|
||||
Assert.True(back);
|
||||
Assert.True(_provider.IsOnPreferredNode(SiteId));
|
||||
|
||||
var beforeA = _stubA.LifecycleCalls;
|
||||
await CallAsync();
|
||||
Assert.Equal(beforeA + 1, _stubA.LifecycleCalls); // next call back on A
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeadlineExceeded_OnNodeA_IsNotRetriedOnNodeB()
|
||||
{
|
||||
// A DeadlineExceeded is ambiguous (a WriteTag/Deploy/Failover may already have executed), so
|
||||
// — unlike Unavailable — it must NOT fail over to B. Modelled by A returning the status
|
||||
// directly, isolating the retry-decision from TestServer's own timeout mechanics.
|
||||
_stubA.StatusToThrow = StatusCode.DeadlineExceeded;
|
||||
|
||||
var ex = await Assert.ThrowsAsync<RpcException>(() => CallAsync());
|
||||
|
||||
Assert.Equal(StatusCode.DeadlineExceeded, ex.StatusCode);
|
||||
Assert.Equal(1, _stubA.LifecycleCalls);
|
||||
Assert.Equal(0, _stubB.LifecycleCalls); // B was never tried
|
||||
Assert.True(_provider.IsOnPreferredNode(SiteId), "a deadline must not flip stickiness");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UnknownSite_Throws_SiteChannelUnavailable()
|
||||
{
|
||||
await Assert.ThrowsAsync<SiteChannelUnavailableException>(
|
||||
() => _provider.ExecuteAsync<LifecycleReply>(
|
||||
"not-configured", (_, _) => Task.FromResult(new LifecycleReply()), CancellationToken.None));
|
||||
}
|
||||
|
||||
private sealed class FixedPskProvider(string key) : ISitePskProvider
|
||||
{
|
||||
public ValueTask<string> GetAsync(string siteId, CancellationToken ct) => new(key);
|
||||
public void Invalidate(string siteId) { }
|
||||
}
|
||||
|
||||
/// <summary>Stub SiteCommandService recording what the client sent and letting a test steer faults.</summary>
|
||||
private sealed class StubSiteCommandService : SiteCommandService.SiteCommandServiceBase
|
||||
{
|
||||
private int _lifecycleCalls;
|
||||
public int LifecycleCalls => Volatile.Read(ref _lifecycleCalls);
|
||||
public string? LastAuthHeader { get; private set; }
|
||||
public string? LastSiteHeader { get; private set; }
|
||||
public bool DeadlineWasSet { get; private set; }
|
||||
public volatile bool ThrowUnavailable;
|
||||
public StatusCode? StatusToThrow;
|
||||
|
||||
public override Task<LifecycleReply> ExecuteLifecycle(LifecycleRequest request, ServerCallContext context)
|
||||
{
|
||||
Interlocked.Increment(ref _lifecycleCalls);
|
||||
LastAuthHeader = context.RequestHeaders
|
||||
.FirstOrDefault(h => h.Key == ControlPlaneCredentials.AuthorizationHeader)?.Value;
|
||||
LastSiteHeader = context.RequestHeaders
|
||||
.FirstOrDefault(h => h.Key == ControlPlaneCredentials.SiteHeader)?.Value;
|
||||
DeadlineWasSet = context.Deadline != DateTime.MaxValue;
|
||||
|
||||
if (ThrowUnavailable)
|
||||
{
|
||||
throw new RpcException(new Status(StatusCode.Unavailable, "node down"));
|
||||
}
|
||||
if (StatusToThrow is { } status)
|
||||
{
|
||||
throw new RpcException(new Status(status, "modelled fault"));
|
||||
}
|
||||
|
||||
return Task.FromResult(SiteCommandDtoMapper.ToLifecycleReply(
|
||||
new InstanceLifecycleResponse("cmd-1", "Site1.Pump1", true, null, DateTimeOffset.UtcNow)));
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user