feat(grpc): site-side ICentralTransport seam + gRPC transport (T1A.3)
Introduce ICentralTransport as the site->central choke point inside SiteCommunicationActor. The seven site->central sends (notification submit/ status, audit + cached-telemetry ingest, reconcile, health, heartbeat) now delegate to an injected transport instead of owning ClusterClient.Send inline. - AkkaCentralTransport: verbatim extraction of today's ClusterClient.Send path, including the exact sender-forwarding that routes central's reply straight back to the waiting Ask. Default when no transport is injected -> behaviour unchanged, existing SiteCommunicationActorTests pass as-is. - GrpcCentralTransport + CentralChannelProvider: dial CentralControlService with sticky failover + background failback (1s-doubling-cap-60s), PSK + x-scadabridge-site via ControlPlaneCredentials, per-call deadlines mirroring today's Ask timeouts. Cross-node retry ONLY on provably-unsent connect failures; never on DeadlineExceeded. Heartbeat stays fire-and-forget. - StaticSitePskProvider: site's single own-key provider (fail-closed). - CommunicationOptions: CentralTransport flag (default Akka) + CentralGrpcEndpoints (validator: required when transport=Grpc). Host selects the impl; the ClusterClient is created only on the Akka path. Tests: actor-with-fake-transport (7 delegations + fault routing + heartbeat no-fault), AkkaCentralTransport sender-forwarding, GrpcCentralTransport over in-process TestServer (failover flip, sticky, failback, PSK+header, deadline, no-retry-on-deadline), validator. Communication.Tests 371 green, Host.Tests 391 green; the three above-seam suites pass unmodified.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,67 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.Client;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// T1A.3: the regression-prone bit of the Akka transport — that a forwarded
|
||||
/// <see cref="ClusterClient.Send"/> carries the caller's sender, so central's reply routes
|
||||
/// straight back to the waiting Ask rather than to the site communication actor — plus the
|
||||
/// no-ClusterClient-yet fallback that keeps the S&F layer treating the send as transient.
|
||||
/// </summary>
|
||||
public class AkkaCentralTransportTests : TestKit
|
||||
{
|
||||
[Fact]
|
||||
public void SubmitNotification_ForwardsToClusterClient_WithReplyToAsSender()
|
||||
{
|
||||
var transport = new AkkaCentralTransport();
|
||||
var clusterClient = CreateTestProbe();
|
||||
var replyTo = CreateTestProbe();
|
||||
transport.SetCentralClient(clusterClient.Ref);
|
||||
|
||||
var submit = new NotificationSubmit(
|
||||
"notif-1", "Operators", "S", "B", "site1", null, null, DateTimeOffset.UtcNow);
|
||||
transport.SubmitNotification(submit, replyTo.Ref);
|
||||
|
||||
// The ClusterClient receives a Send addressed to the central actor...
|
||||
var send = clusterClient.ExpectMsg<ClusterClient.Send>();
|
||||
Assert.Equal("/user/central-communication", send.Path);
|
||||
Assert.IsType<NotificationSubmit>(send.Message);
|
||||
|
||||
// ...and replying to it lands at replyTo, proving the sender was forwarded (not the
|
||||
// transport / actor). This is the routing the waiting Ask relies on.
|
||||
clusterClient.Reply(new NotificationSubmitAck("notif-1", Accepted: true, Error: null));
|
||||
replyTo.ExpectMsg<NotificationSubmitAck>(ack => ack.NotificationId == "notif-1" && ack.Accepted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SubmitNotification_WithNoClusterClient_RepliesNonAcceptedToReplyTo()
|
||||
{
|
||||
var transport = new AkkaCentralTransport();
|
||||
var replyTo = CreateTestProbe();
|
||||
|
||||
transport.SubmitNotification(
|
||||
new NotificationSubmit("notif-2", "Operators", "S", "B", "site1", null, null, DateTimeOffset.UtcNow),
|
||||
replyTo.Ref);
|
||||
|
||||
replyTo.ExpectMsg<NotificationSubmitAck>(ack => ack.NotificationId == "notif-2" && !ack.Accepted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IngestAuditEvents_WithNoClusterClient_FaultsTheReplyTo()
|
||||
{
|
||||
// The audit drain treats a faulted Ask as transient and keeps rows Pending — so the
|
||||
// no-client path must be a Status.Failure, not a silent drop.
|
||||
var transport = new AkkaCentralTransport();
|
||||
var replyTo = CreateTestProbe();
|
||||
|
||||
transport.IngestAuditEvents(
|
||||
new Commons.Messages.Audit.IngestAuditEventsCommand(new List<ZB.MOM.WW.Audit.AuditEvent>()),
|
||||
replyTo.Ref);
|
||||
|
||||
replyTo.ExpectMsg<Status.Failure>();
|
||||
}
|
||||
}
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// T1A.3: the site communication actor delegates each of the seven site→central sends to the
|
||||
/// injected <see cref="ICentralTransport"/>, preserving the current <c>Sender</c> as the reply
|
||||
/// target — and a transport failure surfaces to that sender exactly as the S&F / audit / health
|
||||
/// layers already expect, while a heartbeat transport fault never faults the actor.
|
||||
/// </summary>
|
||||
public class SiteCommunicationActorTransportTests : TestKit
|
||||
{
|
||||
private readonly CommunicationOptions _options = new();
|
||||
|
||||
private (IActorRef actor, ICentralTransport transport) NewActor()
|
||||
{
|
||||
var transport = Substitute.For<ICentralTransport>();
|
||||
var dmProbe = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(Props.Create(() =>
|
||||
new SiteCommunicationActor("site1", _options, dmProbe.Ref, () => false, null, transport)));
|
||||
return (actor, transport);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NotificationSubmit_DelegatesToTransport_WithSenderAsReplyTo()
|
||||
{
|
||||
var (actor, transport) = NewActor();
|
||||
var submit = new NotificationSubmit(
|
||||
"notif-1", "Operators", "Subj", "Body", "site1", "inst1", "alarmScript", DateTimeOffset.UtcNow);
|
||||
|
||||
actor.Tell(submit, TestActor);
|
||||
|
||||
AwaitAssert(() => transport.Received(1).SubmitNotification(
|
||||
Arg.Is<NotificationSubmit>(m => m.NotificationId == "notif-1"), Arg.Is(TestActor)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NotificationStatusQuery_DelegatesToTransport_WithSenderAsReplyTo()
|
||||
{
|
||||
var (actor, transport) = NewActor();
|
||||
|
||||
actor.Tell(new NotificationStatusQuery("corr-1", "notif-1"), TestActor);
|
||||
|
||||
AwaitAssert(() => transport.Received(1).QueryNotificationStatus(
|
||||
Arg.Is<NotificationStatusQuery>(m => m.NotificationId == "notif-1"), Arg.Is(TestActor)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IngestAuditEvents_DelegatesToTransport_WithSenderAsReplyTo()
|
||||
{
|
||||
var (actor, transport) = NewActor();
|
||||
|
||||
actor.Tell(new IngestAuditEventsCommand(new List<AuditEvent>()), TestActor);
|
||||
|
||||
AwaitAssert(() => transport.Received(1).IngestAuditEvents(Arg.Any<IngestAuditEventsCommand>(), Arg.Is(TestActor)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IngestCachedTelemetry_DelegatesToTransport_WithSenderAsReplyTo()
|
||||
{
|
||||
var (actor, transport) = NewActor();
|
||||
|
||||
actor.Tell(new IngestCachedTelemetryCommand(new List<CachedTelemetryEntry>()), TestActor);
|
||||
|
||||
AwaitAssert(() => transport.Received(1).IngestCachedTelemetry(Arg.Any<IngestCachedTelemetryCommand>(), Arg.Is(TestActor)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReconcileSite_DelegatesToTransport_WithSenderAsReplyTo()
|
||||
{
|
||||
var (actor, transport) = NewActor();
|
||||
|
||||
actor.Tell(
|
||||
new ReconcileSiteRequest("site1", "node-a", new Dictionary<string, string>()), TestActor);
|
||||
|
||||
AwaitAssert(() => transport.Received(1).ReconcileSite(
|
||||
Arg.Is<ReconcileSiteRequest>(m => m.NodeId == "node-a"), Arg.Is(TestActor)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReportSiteHealth_DelegatesToTransport_WithSenderAsReplyTo()
|
||||
{
|
||||
var (actor, transport) = NewActor();
|
||||
var report = MinimalHealthReport(sequence: 7);
|
||||
|
||||
actor.Tell(report, TestActor);
|
||||
|
||||
AwaitAssert(() => transport.Received(1).ReportSiteHealth(
|
||||
Arg.Is<SiteHealthReport>(m => m.SequenceNumber == 7), Arg.Is(TestActor)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TransportFailureReply_RoutesBackToTheWaitingSender()
|
||||
{
|
||||
// The seam preserves the reply-routing the S&F layer depends on: when the transport
|
||||
// answers the captured replyTo with a Status.Failure (its transient-failure signal on a
|
||||
// non-OK status), that failure reaches the original sender — here the test actor — so the
|
||||
// waiting Ask faults, exactly as it did on the ClusterClient path.
|
||||
var transport = Substitute.For<ICentralTransport>();
|
||||
transport
|
||||
.When(t => t.SubmitNotification(Arg.Any<NotificationSubmit>(), Arg.Any<IActorRef>()))
|
||||
.Do(ci => ci.Arg<IActorRef>().Tell(
|
||||
new Status.Failure(new InvalidOperationException("central unavailable"))));
|
||||
|
||||
var dmProbe = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(Props.Create(() =>
|
||||
new SiteCommunicationActor("site1", _options, dmProbe.Ref, () => false, null, transport)));
|
||||
|
||||
actor.Tell(new NotificationSubmit(
|
||||
"notif-x", "Operators", "S", "B", "site1", null, null, DateTimeOffset.UtcNow), TestActor);
|
||||
|
||||
var failure = ExpectMsg<Status.Failure>();
|
||||
Assert.IsType<InvalidOperationException>(failure.Cause);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HeartbeatTransportThrow_DoesNotFaultTheActor()
|
||||
{
|
||||
// A transport whose SendHeartbeat throws must not fault the actor — heartbeats are
|
||||
// fire-and-forget and their failure is swallowed. Prove the actor still serves messages
|
||||
// after a heartbeat that threw.
|
||||
var transport = Substitute.For<ICentralTransport>();
|
||||
transport
|
||||
.When(t => t.SendHeartbeat(Arg.Any<HeartbeatMessage>(), Arg.Any<IActorRef>()))
|
||||
.Do(_ => throw new InvalidOperationException("boom"));
|
||||
|
||||
var dmProbe = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(Props.Create(() =>
|
||||
new SiteCommunicationActor(
|
||||
"site1",
|
||||
new CommunicationOptions { ApplicationHeartbeatInterval = TimeSpan.FromMilliseconds(50) },
|
||||
dmProbe.Ref, () => true, null, transport)));
|
||||
|
||||
// Let the heartbeat timer fire a few times (each throws inside the transport).
|
||||
Thread.Sleep(250);
|
||||
|
||||
// The actor is still alive and delegating: a subsequent send is handled normally.
|
||||
actor.Tell(new NotificationSubmit(
|
||||
"after-heartbeat", "Operators", "S", "B", "site1", null, null, DateTimeOffset.UtcNow), TestActor);
|
||||
AwaitAssert(() => transport.Received(1).SubmitNotification(
|
||||
Arg.Is<NotificationSubmit>(m => m.NotificationId == "after-heartbeat"), Arg.Is(TestActor)));
|
||||
}
|
||||
|
||||
private static SiteHealthReport MinimalHealthReport(long sequence) => new(
|
||||
SiteId: "site1",
|
||||
SequenceNumber: sequence,
|
||||
ReportTimestamp: DateTimeOffset.UtcNow,
|
||||
DataConnectionStatuses: new Dictionary<string, ConnectionHealth>(),
|
||||
TagResolutionCounts: new Dictionary<string, TagResolutionStatus>(),
|
||||
ScriptErrorCount: 0,
|
||||
AlarmEvaluationErrorCount: 0,
|
||||
StoreAndForwardBufferDepths: new Dictionary<string, int>(),
|
||||
DeadLetterCount: 0,
|
||||
DeployedInstanceCount: 0,
|
||||
EnabledInstanceCount: 0,
|
||||
DisabledInstanceCount: 0);
|
||||
}
|
||||
@@ -104,4 +104,57 @@ public class CommunicationOptionsValidatorTests
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("LiveAlarmCachePublishCoalesce", result.FailureMessage);
|
||||
}
|
||||
|
||||
// ── T1A.3: gRPC central transport endpoints (required only when selected) ────
|
||||
|
||||
[Fact]
|
||||
public void GrpcTransport_WithNoEndpoints_IsRejected()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions
|
||||
{
|
||||
CentralTransport = CentralTransportMode.Grpc,
|
||||
CentralGrpcEndpoints = new List<string>(),
|
||||
});
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("CentralGrpcEndpoints", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GrpcTransport_WithBlankEndpoint_IsRejected()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions
|
||||
{
|
||||
CentralTransport = CentralTransportMode.Grpc,
|
||||
CentralGrpcEndpoints = new List<string> { " " },
|
||||
});
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("CentralGrpcEndpoints", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GrpcTransport_WithEndpoints_IsValid()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions
|
||||
{
|
||||
CentralTransport = CentralTransportMode.Grpc,
|
||||
CentralGrpcEndpoints = new List<string>
|
||||
{
|
||||
"http://scadabridge-central-a:8083",
|
||||
"http://scadabridge-central-b:8083",
|
||||
},
|
||||
});
|
||||
Assert.True(result.Succeeded, result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AkkaTransport_IgnoresMissingGrpcEndpoints()
|
||||
{
|
||||
// The default transport must not be forced to declare gRPC endpoints it never dials.
|
||||
var result = Validate(new CommunicationOptions
|
||||
{
|
||||
CentralTransport = CentralTransportMode.Akka,
|
||||
CentralGrpcEndpoints = new List<string>(),
|
||||
});
|
||||
Assert.True(result.Succeeded, result.FailureMessage);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user