feat(grpc): site-side ICentralTransport seam + gRPC transport (T1A.3)
Introduce ICentralTransport as the site->central choke point inside SiteCommunicationActor. The seven site->central sends (notification submit/ status, audit + cached-telemetry ingest, reconcile, health, heartbeat) now delegate to an injected transport instead of owning ClusterClient.Send inline. - AkkaCentralTransport: verbatim extraction of today's ClusterClient.Send path, including the exact sender-forwarding that routes central's reply straight back to the waiting Ask. Default when no transport is injected -> behaviour unchanged, existing SiteCommunicationActorTests pass as-is. - GrpcCentralTransport + CentralChannelProvider: dial CentralControlService with sticky failover + background failback (1s-doubling-cap-60s), PSK + x-scadabridge-site via ControlPlaneCredentials, per-call deadlines mirroring today's Ask timeouts. Cross-node retry ONLY on provably-unsent connect failures; never on DeadlineExceeded. Heartbeat stays fire-and-forget. - StaticSitePskProvider: site's single own-key provider (fail-closed). - CommunicationOptions: CentralTransport flag (default Akka) + CentralGrpcEndpoints (validator: required when transport=Grpc). Host selects the impl; the ClusterClient is created only on the Akka path. Tests: actor-with-fake-transport (7 delegations + fault routing + heartbeat no-fault), AkkaCentralTransport sender-forwarding, GrpcCentralTransport over in-process TestServer (failover flip, sticky, failback, PSK+header, deadline, no-retry-on-deadline), validator. Communication.Tests 371 green, Host.Tests 391 green; the three above-seam suites pass unmodified.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,67 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.Client;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// T1A.3: the regression-prone bit of the Akka transport — that a forwarded
|
||||
/// <see cref="ClusterClient.Send"/> carries the caller's sender, so central's reply routes
|
||||
/// straight back to the waiting Ask rather than to the site communication actor — plus the
|
||||
/// no-ClusterClient-yet fallback that keeps the S&F layer treating the send as transient.
|
||||
/// </summary>
|
||||
public class AkkaCentralTransportTests : TestKit
|
||||
{
|
||||
[Fact]
|
||||
public void SubmitNotification_ForwardsToClusterClient_WithReplyToAsSender()
|
||||
{
|
||||
var transport = new AkkaCentralTransport();
|
||||
var clusterClient = CreateTestProbe();
|
||||
var replyTo = CreateTestProbe();
|
||||
transport.SetCentralClient(clusterClient.Ref);
|
||||
|
||||
var submit = new NotificationSubmit(
|
||||
"notif-1", "Operators", "S", "B", "site1", null, null, DateTimeOffset.UtcNow);
|
||||
transport.SubmitNotification(submit, replyTo.Ref);
|
||||
|
||||
// The ClusterClient receives a Send addressed to the central actor...
|
||||
var send = clusterClient.ExpectMsg<ClusterClient.Send>();
|
||||
Assert.Equal("/user/central-communication", send.Path);
|
||||
Assert.IsType<NotificationSubmit>(send.Message);
|
||||
|
||||
// ...and replying to it lands at replyTo, proving the sender was forwarded (not the
|
||||
// transport / actor). This is the routing the waiting Ask relies on.
|
||||
clusterClient.Reply(new NotificationSubmitAck("notif-1", Accepted: true, Error: null));
|
||||
replyTo.ExpectMsg<NotificationSubmitAck>(ack => ack.NotificationId == "notif-1" && ack.Accepted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SubmitNotification_WithNoClusterClient_RepliesNonAcceptedToReplyTo()
|
||||
{
|
||||
var transport = new AkkaCentralTransport();
|
||||
var replyTo = CreateTestProbe();
|
||||
|
||||
transport.SubmitNotification(
|
||||
new NotificationSubmit("notif-2", "Operators", "S", "B", "site1", null, null, DateTimeOffset.UtcNow),
|
||||
replyTo.Ref);
|
||||
|
||||
replyTo.ExpectMsg<NotificationSubmitAck>(ack => ack.NotificationId == "notif-2" && !ack.Accepted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IngestAuditEvents_WithNoClusterClient_FaultsTheReplyTo()
|
||||
{
|
||||
// The audit drain treats a faulted Ask as transient and keeps rows Pending — so the
|
||||
// no-client path must be a Status.Failure, not a silent drop.
|
||||
var transport = new AkkaCentralTransport();
|
||||
var replyTo = CreateTestProbe();
|
||||
|
||||
transport.IngestAuditEvents(
|
||||
new Commons.Messages.Audit.IngestAuditEventsCommand(new List<ZB.MOM.WW.Audit.AuditEvent>()),
|
||||
replyTo.Ref);
|
||||
|
||||
replyTo.ExpectMsg<Status.Failure>();
|
||||
}
|
||||
}
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// T1A.3: the site communication actor delegates each of the seven site→central sends to the
|
||||
/// injected <see cref="ICentralTransport"/>, preserving the current <c>Sender</c> as the reply
|
||||
/// target — and a transport failure surfaces to that sender exactly as the S&F / audit / health
|
||||
/// layers already expect, while a heartbeat transport fault never faults the actor.
|
||||
/// </summary>
|
||||
public class SiteCommunicationActorTransportTests : TestKit
|
||||
{
|
||||
private readonly CommunicationOptions _options = new();
|
||||
|
||||
private (IActorRef actor, ICentralTransport transport) NewActor()
|
||||
{
|
||||
var transport = Substitute.For<ICentralTransport>();
|
||||
var dmProbe = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(Props.Create(() =>
|
||||
new SiteCommunicationActor("site1", _options, dmProbe.Ref, () => false, null, transport)));
|
||||
return (actor, transport);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NotificationSubmit_DelegatesToTransport_WithSenderAsReplyTo()
|
||||
{
|
||||
var (actor, transport) = NewActor();
|
||||
var submit = new NotificationSubmit(
|
||||
"notif-1", "Operators", "Subj", "Body", "site1", "inst1", "alarmScript", DateTimeOffset.UtcNow);
|
||||
|
||||
actor.Tell(submit, TestActor);
|
||||
|
||||
AwaitAssert(() => transport.Received(1).SubmitNotification(
|
||||
Arg.Is<NotificationSubmit>(m => m.NotificationId == "notif-1"), Arg.Is(TestActor)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NotificationStatusQuery_DelegatesToTransport_WithSenderAsReplyTo()
|
||||
{
|
||||
var (actor, transport) = NewActor();
|
||||
|
||||
actor.Tell(new NotificationStatusQuery("corr-1", "notif-1"), TestActor);
|
||||
|
||||
AwaitAssert(() => transport.Received(1).QueryNotificationStatus(
|
||||
Arg.Is<NotificationStatusQuery>(m => m.NotificationId == "notif-1"), Arg.Is(TestActor)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IngestAuditEvents_DelegatesToTransport_WithSenderAsReplyTo()
|
||||
{
|
||||
var (actor, transport) = NewActor();
|
||||
|
||||
actor.Tell(new IngestAuditEventsCommand(new List<AuditEvent>()), TestActor);
|
||||
|
||||
AwaitAssert(() => transport.Received(1).IngestAuditEvents(Arg.Any<IngestAuditEventsCommand>(), Arg.Is(TestActor)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IngestCachedTelemetry_DelegatesToTransport_WithSenderAsReplyTo()
|
||||
{
|
||||
var (actor, transport) = NewActor();
|
||||
|
||||
actor.Tell(new IngestCachedTelemetryCommand(new List<CachedTelemetryEntry>()), TestActor);
|
||||
|
||||
AwaitAssert(() => transport.Received(1).IngestCachedTelemetry(Arg.Any<IngestCachedTelemetryCommand>(), Arg.Is(TestActor)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReconcileSite_DelegatesToTransport_WithSenderAsReplyTo()
|
||||
{
|
||||
var (actor, transport) = NewActor();
|
||||
|
||||
actor.Tell(
|
||||
new ReconcileSiteRequest("site1", "node-a", new Dictionary<string, string>()), TestActor);
|
||||
|
||||
AwaitAssert(() => transport.Received(1).ReconcileSite(
|
||||
Arg.Is<ReconcileSiteRequest>(m => m.NodeId == "node-a"), Arg.Is(TestActor)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReportSiteHealth_DelegatesToTransport_WithSenderAsReplyTo()
|
||||
{
|
||||
var (actor, transport) = NewActor();
|
||||
var report = MinimalHealthReport(sequence: 7);
|
||||
|
||||
actor.Tell(report, TestActor);
|
||||
|
||||
AwaitAssert(() => transport.Received(1).ReportSiteHealth(
|
||||
Arg.Is<SiteHealthReport>(m => m.SequenceNumber == 7), Arg.Is(TestActor)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TransportFailureReply_RoutesBackToTheWaitingSender()
|
||||
{
|
||||
// The seam preserves the reply-routing the S&F layer depends on: when the transport
|
||||
// answers the captured replyTo with a Status.Failure (its transient-failure signal on a
|
||||
// non-OK status), that failure reaches the original sender — here the test actor — so the
|
||||
// waiting Ask faults, exactly as it did on the ClusterClient path.
|
||||
var transport = Substitute.For<ICentralTransport>();
|
||||
transport
|
||||
.When(t => t.SubmitNotification(Arg.Any<NotificationSubmit>(), Arg.Any<IActorRef>()))
|
||||
.Do(ci => ci.Arg<IActorRef>().Tell(
|
||||
new Status.Failure(new InvalidOperationException("central unavailable"))));
|
||||
|
||||
var dmProbe = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(Props.Create(() =>
|
||||
new SiteCommunicationActor("site1", _options, dmProbe.Ref, () => false, null, transport)));
|
||||
|
||||
actor.Tell(new NotificationSubmit(
|
||||
"notif-x", "Operators", "S", "B", "site1", null, null, DateTimeOffset.UtcNow), TestActor);
|
||||
|
||||
var failure = ExpectMsg<Status.Failure>();
|
||||
Assert.IsType<InvalidOperationException>(failure.Cause);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HeartbeatTransportThrow_DoesNotFaultTheActor()
|
||||
{
|
||||
// A transport whose SendHeartbeat throws must not fault the actor — heartbeats are
|
||||
// fire-and-forget and their failure is swallowed. Prove the actor still serves messages
|
||||
// after a heartbeat that threw.
|
||||
var transport = Substitute.For<ICentralTransport>();
|
||||
transport
|
||||
.When(t => t.SendHeartbeat(Arg.Any<HeartbeatMessage>(), Arg.Any<IActorRef>()))
|
||||
.Do(_ => throw new InvalidOperationException("boom"));
|
||||
|
||||
var dmProbe = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(Props.Create(() =>
|
||||
new SiteCommunicationActor(
|
||||
"site1",
|
||||
new CommunicationOptions { ApplicationHeartbeatInterval = TimeSpan.FromMilliseconds(50) },
|
||||
dmProbe.Ref, () => true, null, transport)));
|
||||
|
||||
// Let the heartbeat timer fire a few times (each throws inside the transport).
|
||||
Thread.Sleep(250);
|
||||
|
||||
// The actor is still alive and delegating: a subsequent send is handled normally.
|
||||
actor.Tell(new NotificationSubmit(
|
||||
"after-heartbeat", "Operators", "S", "B", "site1", null, null, DateTimeOffset.UtcNow), TestActor);
|
||||
AwaitAssert(() => transport.Received(1).SubmitNotification(
|
||||
Arg.Is<NotificationSubmit>(m => m.NotificationId == "after-heartbeat"), Arg.Is(TestActor)));
|
||||
}
|
||||
|
||||
private static SiteHealthReport MinimalHealthReport(long sequence) => new(
|
||||
SiteId: "site1",
|
||||
SequenceNumber: sequence,
|
||||
ReportTimestamp: DateTimeOffset.UtcNow,
|
||||
DataConnectionStatuses: new Dictionary<string, ConnectionHealth>(),
|
||||
TagResolutionCounts: new Dictionary<string, TagResolutionStatus>(),
|
||||
ScriptErrorCount: 0,
|
||||
AlarmEvaluationErrorCount: 0,
|
||||
StoreAndForwardBufferDepths: new Dictionary<string, int>(),
|
||||
DeadLetterCount: 0,
|
||||
DeployedInstanceCount: 0,
|
||||
EnabledInstanceCount: 0,
|
||||
DisabledInstanceCount: 0);
|
||||
}
|
||||
@@ -104,4 +104,57 @@ public class CommunicationOptionsValidatorTests
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("LiveAlarmCachePublishCoalesce", result.FailureMessage);
|
||||
}
|
||||
|
||||
// ── T1A.3: gRPC central transport endpoints (required only when selected) ────
|
||||
|
||||
[Fact]
|
||||
public void GrpcTransport_WithNoEndpoints_IsRejected()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions
|
||||
{
|
||||
CentralTransport = CentralTransportMode.Grpc,
|
||||
CentralGrpcEndpoints = new List<string>(),
|
||||
});
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("CentralGrpcEndpoints", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GrpcTransport_WithBlankEndpoint_IsRejected()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions
|
||||
{
|
||||
CentralTransport = CentralTransportMode.Grpc,
|
||||
CentralGrpcEndpoints = new List<string> { " " },
|
||||
});
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("CentralGrpcEndpoints", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GrpcTransport_WithEndpoints_IsValid()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions
|
||||
{
|
||||
CentralTransport = CentralTransportMode.Grpc,
|
||||
CentralGrpcEndpoints = new List<string>
|
||||
{
|
||||
"http://scadabridge-central-a:8083",
|
||||
"http://scadabridge-central-b:8083",
|
||||
},
|
||||
});
|
||||
Assert.True(result.Succeeded, result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AkkaTransport_IgnoresMissingGrpcEndpoints()
|
||||
{
|
||||
// The default transport must not be forced to declare gRPC endpoints it never dials.
|
||||
var result = Validate(new CommunicationOptions
|
||||
{
|
||||
CentralTransport = CentralTransportMode.Akka,
|
||||
CentralGrpcEndpoints = new List<string>(),
|
||||
});
|
||||
Assert.True(result.Succeeded, result.FailureMessage);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,373 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Akka.Actor;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.TestHost;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// T1A.3: <see cref="GrpcCentralTransport"/> + <see cref="CentralChannelProvider"/> over a real
|
||||
/// gRPC stack (two in-process <see cref="TestServer"/> central nodes, the real
|
||||
/// <see cref="CentralControlGrpcService"/> and <see cref="CentralControlAuthInterceptor"/>). Proves
|
||||
/// the sticky failover/failback policy, the PSK + site-header attachment, the per-call deadline,
|
||||
/// and — the hard rule — no cross-node retry on <c>DeadlineExceeded</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A "down" node is modelled by a <see cref="ToggleHandler"/> that throws before reaching the
|
||||
/// TestServer, so BOTH the unary call and the failback <c>Heartbeat</c> probe see it as
|
||||
/// <c>Unavailable</c> — the honest shape of a refused connection, and the only class the transport
|
||||
/// fails over on. Readiness is always set, so a node that is "up" answers everything.
|
||||
/// </remarks>
|
||||
public class GrpcCentralTransportTests : IAsyncLifetime
|
||||
{
|
||||
private const string SiteA = "site-a";
|
||||
private const string SiteAKey = "site-a-preshared-key";
|
||||
private const string EndpointA = "http://central-a/";
|
||||
private const string EndpointB = "http://central-b/";
|
||||
|
||||
private ActorSystem _system = null!;
|
||||
private CentralNode _nodeA = null!;
|
||||
private CentralNode _nodeB = null!;
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
_system = ActorSystem.Create("grpc-central-transport-test");
|
||||
_nodeA = await CentralNode.StartAsync(_system, "A", SiteA, SiteAKey, repliesToSubmit: true);
|
||||
_nodeB = await CentralNode.StartAsync(_system, "B", SiteA, SiteAKey, repliesToSubmit: true);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
await _nodeA.DisposeAsync();
|
||||
await _nodeB.DisposeAsync();
|
||||
await _system.Terminate();
|
||||
}
|
||||
|
||||
private CentralChannelProvider NewProvider(string? pskKey = SiteAKey) => new(
|
||||
new[] { EndpointA, EndpointB },
|
||||
new FixedPskProvider(pskKey),
|
||||
SiteA,
|
||||
new CommunicationOptions(),
|
||||
NullLogger<CentralChannelProvider>.Instance,
|
||||
handlerFactory: HandlerFor,
|
||||
probeDeadline: TimeSpan.FromSeconds(2),
|
||||
backoffBase: TimeSpan.FromMilliseconds(50),
|
||||
backoffCap: TimeSpan.FromMilliseconds(200));
|
||||
|
||||
private HttpMessageHandler HandlerFor(string endpoint) => endpoint == EndpointA
|
||||
? new ToggleHandler(_nodeA.Server.CreateHandler(), () => _nodeA.IsUp)
|
||||
: new ToggleHandler(_nodeB.Server.CreateHandler(), () => _nodeB.IsUp);
|
||||
|
||||
private GrpcCentralTransport NewTransport(CentralChannelProvider provider, CommunicationOptions? options = null)
|
||||
=> new(provider, options ?? new CommunicationOptions(), NullLogger<GrpcCentralTransport>.Instance);
|
||||
|
||||
[Fact]
|
||||
public async Task HappyPath_ReachesThePreferredNode_AndRoutesTheAckBack()
|
||||
{
|
||||
using var provider = NewProvider();
|
||||
var transport = NewTransport(provider);
|
||||
var inbox = new Capture(_system);
|
||||
|
||||
transport.SubmitNotification(NewSubmit("n1"), inbox.Ref);
|
||||
|
||||
var ack = Assert.IsType<NotificationSubmitAck>(inbox.Receive(TimeSpan.FromSeconds(5)));
|
||||
Assert.True(ack.Accepted);
|
||||
Assert.Equal("n1", ack.NotificationId);
|
||||
Assert.Equal(0, provider.CurrentIndex); // stayed on preferred
|
||||
Assert.Equal(1, _nodeA.SubmitCount);
|
||||
Assert.Equal(0, _nodeB.SubmitCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sticky_StaysOnThePreferredNode_WhileHealthy()
|
||||
{
|
||||
using var provider = NewProvider();
|
||||
var transport = NewTransport(provider);
|
||||
|
||||
for (var i = 0; i < 4; i++)
|
||||
{
|
||||
var inbox = new Capture(_system);
|
||||
transport.SubmitNotification(NewSubmit($"n{i}"), inbox.Ref);
|
||||
Assert.IsType<NotificationSubmitAck>(inbox.Receive(TimeSpan.FromSeconds(5)));
|
||||
}
|
||||
|
||||
Assert.Equal(0, provider.CurrentIndex);
|
||||
Assert.Equal(4, _nodeA.SubmitCount);
|
||||
Assert.Equal(0, _nodeB.SubmitCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Failover_FlipsToThePeer_WhenThePreferredIsUnavailable()
|
||||
{
|
||||
using var provider = NewProvider();
|
||||
var transport = NewTransport(provider);
|
||||
_nodeA.IsUp = false; // preferred refuses connections
|
||||
|
||||
var inbox = new Capture(_system);
|
||||
transport.SubmitNotification(NewSubmit("n1"), inbox.Ref);
|
||||
|
||||
var ack = Assert.IsType<NotificationSubmitAck>(inbox.Receive(TimeSpan.FromSeconds(5)));
|
||||
Assert.True(ack.Accepted);
|
||||
Assert.Equal(1, provider.CurrentIndex); // flipped to the peer
|
||||
Assert.Equal(0, _nodeA.SubmitCount);
|
||||
Assert.Equal(1, _nodeB.SubmitCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Failback_ReturnsToThePreferred_OnceItIsReachableAgain()
|
||||
{
|
||||
using var provider = NewProvider();
|
||||
var transport = NewTransport(provider);
|
||||
|
||||
// Take the preferred down and drive one call so we flip to the peer + arm the failback probe.
|
||||
_nodeA.IsUp = false;
|
||||
var inbox = new Capture(_system);
|
||||
transport.SubmitNotification(NewSubmit("n1"), inbox.Ref);
|
||||
Assert.IsType<NotificationSubmitAck>(inbox.Receive(TimeSpan.FromSeconds(5)));
|
||||
Assert.Equal(1, provider.CurrentIndex);
|
||||
|
||||
// Bring the preferred back; the background probe should fail us back within a few backoffs.
|
||||
_nodeA.IsUp = true;
|
||||
await WaitUntil(() => provider.CurrentIndex == 0, TimeSpan.FromSeconds(5));
|
||||
Assert.Equal(0, provider.CurrentIndex);
|
||||
|
||||
// New calls resume on the preferred node.
|
||||
var inbox2 = new Capture(_system);
|
||||
transport.SubmitNotification(NewSubmit("n2"), inbox2.Ref);
|
||||
Assert.IsType<NotificationSubmitAck>(inbox2.Receive(TimeSpan.FromSeconds(5)));
|
||||
Assert.True(_nodeA.SubmitCount >= 1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PskAndSiteHeader_AreAttached_SoTheGatedCallReachesTheService()
|
||||
{
|
||||
// The service is gated by CentralControlAuthInterceptor; a call that reaches it (and gets
|
||||
// Accepted) proves both the bearer PSK and the x-scadabridge-site header were attached.
|
||||
using var provider = NewProvider(pskKey: SiteAKey);
|
||||
var transport = NewTransport(provider);
|
||||
var inbox = new Capture(_system);
|
||||
|
||||
transport.SubmitNotification(NewSubmit("n1"), inbox.Ref);
|
||||
|
||||
var ack = Assert.IsType<NotificationSubmitAck>(inbox.Receive(TimeSpan.FromSeconds(5)));
|
||||
Assert.True(ack.Accepted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WrongPsk_IsRejected_AndNotRetriedOnThePeer()
|
||||
{
|
||||
// PermissionDenied is not a connect failure — the transport surfaces it as a transient
|
||||
// Status.Failure without flipping to the peer.
|
||||
using var provider = NewProvider(pskKey: "the-wrong-key");
|
||||
var transport = NewTransport(provider);
|
||||
var inbox = new Capture(_system);
|
||||
|
||||
transport.SubmitNotification(NewSubmit("n1"), inbox.Ref);
|
||||
|
||||
Assert.IsType<Status.Failure>(inbox.Receive(TimeSpan.FromSeconds(5)));
|
||||
Assert.Equal(0, provider.CurrentIndex); // no flip
|
||||
Assert.Equal(0, _nodeB.SubmitCount); // peer never tried
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeadlineExceeded_IsNotRetriedOnThePeer()
|
||||
{
|
||||
// THE hard rule. Node A is UP but never replies, so the call deadlines. The transport must
|
||||
// surface Status.Failure and must NOT try node B (the call may already have executed).
|
||||
_nodeA.SetBlackHole();
|
||||
var shortDeadline = new CommunicationOptions { NotificationForwardTimeout = TimeSpan.FromMilliseconds(300) };
|
||||
|
||||
using var provider = NewProvider();
|
||||
var transport = NewTransport(provider, shortDeadline);
|
||||
var inbox = new Capture(_system);
|
||||
|
||||
transport.SubmitNotification(NewSubmit("n1"), inbox.Ref);
|
||||
|
||||
// A per-call deadline is applied (the call returns fast instead of hanging on the black hole).
|
||||
Assert.IsType<Status.Failure>(inbox.Receive(TimeSpan.FromSeconds(5)));
|
||||
Assert.Equal(0, provider.CurrentIndex); // no failover on a deadline
|
||||
Assert.Equal(0, _nodeB.SubmitCount); // peer never tried
|
||||
}
|
||||
|
||||
private static NotificationSubmit NewSubmit(string id) => new(
|
||||
NotificationId: id,
|
||||
ListName: "ops",
|
||||
Subject: "s",
|
||||
Body: "b",
|
||||
SourceSiteId: SiteA,
|
||||
SourceInstanceId: null,
|
||||
SourceScript: null,
|
||||
SiteEnqueuedAt: DateTimeOffset.UtcNow);
|
||||
|
||||
private static async Task WaitUntil(Func<bool> condition, TimeSpan timeout)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + timeout;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
if (condition())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await Task.Delay(25);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A raw message sink used as the transport's <c>replyTo</c>. Unlike Akka's <c>Inbox</c>, which
|
||||
/// rethrows a <see cref="Status.Failure"/>'s cause on receive, this captures every message
|
||||
/// verbatim so a test can assert on the <see cref="Status.Failure"/> itself.
|
||||
/// </summary>
|
||||
private sealed class Capture
|
||||
{
|
||||
private readonly BlockingCollection<object> _messages = new();
|
||||
|
||||
public Capture(ActorSystem system)
|
||||
{
|
||||
Ref = system.ActorOf(Props.Create(() => new CaptureActor(_messages)));
|
||||
}
|
||||
|
||||
public IActorRef Ref { get; }
|
||||
|
||||
public object Receive(TimeSpan timeout)
|
||||
=> _messages.TryTake(out var message, timeout)
|
||||
? message
|
||||
: throw new TimeoutException("No message captured within the timeout.");
|
||||
|
||||
private sealed class CaptureActor : ReceiveActor
|
||||
{
|
||||
public CaptureActor(BlockingCollection<object> messages) => ReceiveAny(messages.Add);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A gRPC channel handler that throws (a refused connection) while its node is "down".</summary>
|
||||
private sealed class ToggleHandler : DelegatingHandler
|
||||
{
|
||||
private readonly Func<bool> _isUp;
|
||||
|
||||
public ToggleHandler(HttpMessageHandler inner, Func<bool> isUp)
|
||||
{
|
||||
InnerHandler = inner;
|
||||
_isUp = isUp;
|
||||
}
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!_isUp())
|
||||
{
|
||||
throw new HttpRequestException("simulated central node down");
|
||||
}
|
||||
|
||||
return await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FixedPskProvider(string? key) : ISitePskProvider
|
||||
{
|
||||
public ValueTask<string> GetAsync(string siteId, CancellationToken ct)
|
||||
=> key is null ? throw new InvalidOperationException("no key") : new ValueTask<string>(key);
|
||||
|
||||
public void Invalidate(string siteId) { }
|
||||
}
|
||||
|
||||
/// <summary>One in-process central node: TestServer + real service/interceptor + a stub actor.</summary>
|
||||
private sealed class CentralNode : IAsyncDisposable
|
||||
{
|
||||
private IHost _host = null!;
|
||||
private IActorRef _stub = null!;
|
||||
private readonly StubCounters _counters = new();
|
||||
|
||||
public TestServer Server { get; private set; } = null!;
|
||||
public volatile bool IsUp = true;
|
||||
public int SubmitCount => _counters.Submits;
|
||||
|
||||
public static async Task<CentralNode> StartAsync(
|
||||
ActorSystem system, string label, string site, string key, bool repliesToSubmit)
|
||||
{
|
||||
var node = new CentralNode();
|
||||
node._stub = system.ActorOf(
|
||||
Props.Create(() => new StubCentralActor(node._counters, repliesToSubmit)), $"stub-{label}");
|
||||
|
||||
var service = new CentralControlGrpcService(
|
||||
NullLogger<CentralControlGrpcService>.Instance,
|
||||
Options.Create(new CommunicationOptions()));
|
||||
service.SetReady(node._stub);
|
||||
|
||||
var psk = new MapPskProvider(new Dictionary<string, string> { [site] = key });
|
||||
|
||||
node._host = await new HostBuilder()
|
||||
.ConfigureWebHost(web => web
|
||||
.UseTestServer()
|
||||
.ConfigureServices(services =>
|
||||
{
|
||||
services.AddGrpc(o => o.Interceptors.Add<CentralControlAuthInterceptor>());
|
||||
services.AddSingleton<ISitePskProvider>(psk);
|
||||
services.AddSingleton(service);
|
||||
})
|
||||
.Configure(app =>
|
||||
{
|
||||
app.UseRouting();
|
||||
app.UseEndpoints(e => e.MapGrpcService<CentralControlGrpcService>());
|
||||
}))
|
||||
.StartAsync();
|
||||
|
||||
node.Server = node._host.GetTestServer();
|
||||
return node;
|
||||
}
|
||||
|
||||
/// <summary>Switches the node's actor to a black hole that counts but never replies.</summary>
|
||||
public void SetBlackHole() => _counters.BlackHole = true;
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _host.StopAsync();
|
||||
_host.Dispose();
|
||||
}
|
||||
|
||||
private sealed class StubCounters
|
||||
{
|
||||
private int _submits;
|
||||
public int Submits => Volatile.Read(ref _submits);
|
||||
public void IncrementSubmits() => Interlocked.Increment(ref _submits);
|
||||
public volatile bool BlackHole;
|
||||
}
|
||||
|
||||
private sealed class StubCentralActor : ReceiveActor
|
||||
{
|
||||
public StubCentralActor(StubCounters counters, bool repliesToSubmit)
|
||||
{
|
||||
Receive<NotificationSubmit>(msg =>
|
||||
{
|
||||
counters.IncrementSubmits();
|
||||
if (repliesToSubmit && !counters.BlackHole)
|
||||
{
|
||||
Sender.Tell(new NotificationSubmitAck(msg.NotificationId, Accepted: true, Error: null));
|
||||
}
|
||||
});
|
||||
|
||||
// Heartbeat lands here as a Tell (the failback probe); ignore it, no reply expected.
|
||||
ReceiveAny(_ => { });
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class MapPskProvider(IReadOnlyDictionary<string, string> keys) : ISitePskProvider
|
||||
{
|
||||
public ValueTask<string> GetAsync(string siteId, CancellationToken ct)
|
||||
=> keys.TryGetValue(siteId, out var key)
|
||||
? new ValueTask<string>(key)
|
||||
: throw new InvalidOperationException($"no key for '{siteId}'");
|
||||
|
||||
public void Invalidate(string siteId) { }
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user