feat(grpc): site-side ICentralTransport seam + gRPC transport (T1A.3)
Introduce ICentralTransport as the site->central choke point inside SiteCommunicationActor. The seven site->central sends (notification submit/ status, audit + cached-telemetry ingest, reconcile, health, heartbeat) now delegate to an injected transport instead of owning ClusterClient.Send inline. - AkkaCentralTransport: verbatim extraction of today's ClusterClient.Send path, including the exact sender-forwarding that routes central's reply straight back to the waiting Ask. Default when no transport is injected -> behaviour unchanged, existing SiteCommunicationActorTests pass as-is. - GrpcCentralTransport + CentralChannelProvider: dial CentralControlService with sticky failover + background failback (1s-doubling-cap-60s), PSK + x-scadabridge-site via ControlPlaneCredentials, per-call deadlines mirroring today's Ask timeouts. Cross-node retry ONLY on provably-unsent connect failures; never on DeadlineExceeded. Heartbeat stays fire-and-forget. - StaticSitePskProvider: site's single own-key provider (fail-closed). - CommunicationOptions: CentralTransport flag (default Akka) + CentralGrpcEndpoints (validator: required when transport=Grpc). Host selects the impl; the ClusterClient is created only on the Akka path. Tests: actor-with-fake-transport (7 delegations + fault routing + heartbeat no-fault), AkkaCentralTransport sender-forwarding, GrpcCentralTransport over in-process TestServer (failover flip, sticky, failback, PSK+header, deadline, no-retry-on-deadline), validator. Communication.Tests 371 green, Host.Tests 391 green; the three above-seam suites pass unmodified.
This commit is contained in:
Joseph Doherty
@@ -1,6 +1,5 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Akka.Cluster.Tools.Client;
|
||||
using Akka.Event;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
|
||||
@@ -45,10 +44,14 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
|
||||
private readonly IActorRef _deploymentManagerProxy;
|
||||
|
||||
/// <summary>
|
||||
/// ClusterClient reference for sending messages to the central cluster.
|
||||
/// Set via RegisterCentralClient message.
|
||||
/// The site→central transport. Finalized in <see cref="PreStart"/> to the injected instance,
|
||||
/// or a default <see cref="AkkaCentralTransport"/> (ClusterClient) when none is supplied — so
|
||||
/// the seven site→central sends delegate here rather than owning the wire plumbing inline.
|
||||
/// </summary>
|
||||
private IActorRef? _centralClient;
|
||||
private ICentralTransport _transport;
|
||||
|
||||
/// <summary>The transport supplied by the Host (null selects the default Akka transport).</summary>
|
||||
private readonly ICentralTransport? _injectedTransport;
|
||||
|
||||
/// <summary>
|
||||
/// Local actor references for routing specific message patterns.
|
||||
@@ -73,24 +76,36 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
|
||||
/// pass a stub so they do not need to load Akka.Cluster into the <c>TestKit</c>
|
||||
/// ActorSystem.
|
||||
/// </param>
|
||||
/// <param name="transport">
|
||||
/// The site→central transport. <c>null</c> (the default, used by every existing test and by
|
||||
/// the Host's Akka path) selects an <see cref="AkkaCentralTransport"/> over ClusterClient; the
|
||||
/// Host injects a <see cref="Grpc.GrpcCentralTransport"/> when
|
||||
/// <c>ScadaBridge:Communication:CentralTransport</c> is <c>Grpc</c>.
|
||||
/// </param>
|
||||
public SiteCommunicationActor(
|
||||
string siteId,
|
||||
CommunicationOptions options,
|
||||
IActorRef deploymentManagerProxy,
|
||||
Func<bool>? isActiveCheck = null,
|
||||
Func<string, string?>? failOverRole = null)
|
||||
Func<string, string?>? failOverRole = null,
|
||||
ICentralTransport? transport = null)
|
||||
{
|
||||
_siteId = siteId;
|
||||
_options = options;
|
||||
_deploymentManagerProxy = deploymentManagerProxy;
|
||||
_isActiveCheck = isActiveCheck ?? DefaultIsActiveCheck;
|
||||
_failOverRole = failOverRole ?? DefaultFailOverRole;
|
||||
_injectedTransport = transport;
|
||||
// Finalized in PreStart (where _log is usable for the default transport); assigned here
|
||||
// too so the field is definitely-assigned for the constructor's Receive closures.
|
||||
_transport = transport!;
|
||||
|
||||
// Registration
|
||||
// Registration. Feeding the ClusterClient into the transport is a no-op unless the
|
||||
// default/Akka transport is in use — the gRPC transport dials configured endpoints and
|
||||
// never receives this message (the Host does not create a ClusterClient for it).
|
||||
Receive<RegisterCentralClient>(msg =>
|
||||
{
|
||||
_centralClient = msg.Client;
|
||||
_log.Info("Registered central ClusterClient");
|
||||
(_transport as AkkaCentralTransport)?.SetCentralClient(msg.Client);
|
||||
});
|
||||
Receive<RegisterLocalHandler>(HandleRegisterLocalHandler);
|
||||
|
||||
@@ -274,157 +289,33 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
|
||||
// from cluster state, not from who received the message.
|
||||
Receive<TriggerSiteFailover>(HandleTriggerSiteFailover);
|
||||
|
||||
// Notification Outbox: forward a buffered notification submitted by the site
|
||||
// Store-and-Forward Engine to the central cluster. The original Sender (the
|
||||
// S&F forwarder's Ask) is forwarded as the ClusterClient.Send sender so the
|
||||
// NotificationSubmitAck routes straight back to the waiting Ask, not here.
|
||||
Receive<NotificationSubmit>(msg =>
|
||||
{
|
||||
if (_centralClient == null)
|
||||
{
|
||||
// No ClusterClient registered yet (e.g. central contact points not
|
||||
// configured, or registration not yet completed). A non-accepted ack
|
||||
// makes the S&F forwarder treat this as transient and retry later.
|
||||
_log.Warning(
|
||||
"Cannot forward NotificationSubmit {0} — no central ClusterClient registered",
|
||||
msg.NotificationId);
|
||||
Sender.Tell(new NotificationSubmitAck(
|
||||
msg.NotificationId, Accepted: false, Error: "Central ClusterClient not registered"));
|
||||
return;
|
||||
}
|
||||
// The seven site→central sends now delegate to the injected transport (ClusterClient by
|
||||
// default, gRPC when configured). Each handler captures the current Sender as the reply
|
||||
// target so central's reply routes straight back to the waiting Ask, not through this
|
||||
// actor — the exact sender-forwarding the ClusterClient path relied on. The per-message
|
||||
// "no transport / not-accepted" fallbacks live inside the transport now.
|
||||
|
||||
_log.Debug("Forwarding NotificationSubmit {0} to central", msg.NotificationId);
|
||||
_centralClient.Tell(
|
||||
new ClusterClient.Send("/user/central-communication", msg), Sender);
|
||||
});
|
||||
// Notification Outbox: forward a buffered notification (S&F forwarder's Ask → ack back).
|
||||
Receive<NotificationSubmit>(msg => _transport.SubmitNotification(msg, Sender));
|
||||
|
||||
// Notification Outbox: forward a Notify.Status query to the central cluster.
|
||||
// The original Sender (the Notify helper's Ask) is forwarded as the
|
||||
// ClusterClient.Send sender so the NotificationStatusResponse routes straight
|
||||
// back to the waiting Ask, not here.
|
||||
Receive<NotificationStatusQuery>(msg =>
|
||||
{
|
||||
if (_centralClient == null)
|
||||
{
|
||||
// No ClusterClient registered yet. Reply Found: false so Notify.Status
|
||||
// falls back to the site S&F buffer to decide Forwarding vs Unknown.
|
||||
_log.Warning(
|
||||
"Cannot forward NotificationStatusQuery {0} — no central ClusterClient registered",
|
||||
msg.NotificationId);
|
||||
Sender.Tell(new NotificationStatusResponse(
|
||||
msg.CorrelationId, Found: false, Status: "Unknown",
|
||||
RetryCount: 0, LastError: null, DeliveredAt: null));
|
||||
return;
|
||||
}
|
||||
// Notification Outbox: forward a Notify.Status query (Notify helper's Ask → response back).
|
||||
Receive<NotificationStatusQuery>(msg => _transport.QueryNotificationStatus(msg, Sender));
|
||||
|
||||
_log.Debug("Forwarding NotificationStatusQuery {0} to central", msg.NotificationId);
|
||||
_centralClient.Tell(
|
||||
new ClusterClient.Send("/user/central-communication", msg), Sender);
|
||||
});
|
||||
// Audit Log: forward a batch of site-local audit events (telemetry drain's Ask → reply back).
|
||||
Receive<IngestAuditEventsCommand>(msg => _transport.IngestAuditEvents(msg, Sender));
|
||||
|
||||
// Audit Log: forward a batch of site-local audit events to the
|
||||
// central cluster. The site SiteAuditTelemetryActor drains its SQLite
|
||||
// Pending queue through the ClusterClientSiteAuditClient, which Asks
|
||||
// this actor; the original Sender (that Ask) is passed as the
|
||||
// ClusterClient.Send sender so the IngestAuditEventsReply routes
|
||||
// straight back to the waiting Ask, not here. Mirrors NotificationSubmit.
|
||||
Receive<IngestAuditEventsCommand>(msg =>
|
||||
{
|
||||
if (_centralClient == null)
|
||||
{
|
||||
// No ClusterClient registered yet (e.g. central contact points
|
||||
// not configured, or registration not yet completed). Faulting
|
||||
// the Ask makes the SiteAuditTelemetryActor drain loop treat
|
||||
// this as transient and keep the rows Pending for the next tick.
|
||||
_log.Warning(
|
||||
"Cannot forward IngestAuditEventsCommand ({0} events) — no central ClusterClient registered",
|
||||
msg.Events.Count);
|
||||
Sender.Tell(new Status.Failure(
|
||||
new InvalidOperationException("Central ClusterClient not registered")));
|
||||
return;
|
||||
}
|
||||
// Audit Log: forward a batch of combined cached-call telemetry (telemetry drain's Ask → reply back).
|
||||
Receive<IngestCachedTelemetryCommand>(msg => _transport.IngestCachedTelemetry(msg, Sender));
|
||||
|
||||
_log.Debug("Forwarding IngestAuditEventsCommand ({0} events) to central", msg.Events.Count);
|
||||
_centralClient.Tell(
|
||||
new ClusterClient.Send("/user/central-communication", msg), Sender);
|
||||
});
|
||||
// Site startup reconciliation: forward the node's local-inventory request (reconcile Ask → response back).
|
||||
Receive<ReconcileSiteRequest>(msg => _transport.ReconcileSite(msg, Sender));
|
||||
|
||||
// Audit Log: forward a batch of combined cached-call telemetry
|
||||
// packets to the central cluster. Same forward + reply-routing pattern
|
||||
// as IngestAuditEventsCommand; central replies with an
|
||||
// IngestCachedTelemetryReply.
|
||||
Receive<IngestCachedTelemetryCommand>(msg =>
|
||||
{
|
||||
if (_centralClient == null)
|
||||
{
|
||||
_log.Warning(
|
||||
"Cannot forward IngestCachedTelemetryCommand ({0} entries) — no central ClusterClient registered",
|
||||
msg.Entries.Count);
|
||||
Sender.Tell(new Status.Failure(
|
||||
new InvalidOperationException("Central ClusterClient not registered")));
|
||||
return;
|
||||
}
|
||||
|
||||
_log.Debug("Forwarding IngestCachedTelemetryCommand ({0} entries) to central", msg.Entries.Count);
|
||||
_centralClient.Tell(
|
||||
new ClusterClient.Send("/user/central-communication", msg), Sender);
|
||||
});
|
||||
|
||||
// Site startup reconciliation: forward the node's local-inventory
|
||||
// ReconcileSiteRequest to the central cluster. The original Sender (the
|
||||
// SiteReconciliationActor's Ask) is passed as the ClusterClient.Send sender so
|
||||
// the ReconcileSiteResponse routes straight back to the waiting Ask, not here.
|
||||
// Mirrors IngestAuditEventsCommand.
|
||||
Receive<ReconcileSiteRequest>(msg =>
|
||||
{
|
||||
if (_centralClient == null)
|
||||
{
|
||||
// No ClusterClient registered yet (e.g. central contact points not
|
||||
// configured, or registration not yet completed). Faulting the Ask makes
|
||||
// the SiteReconciliationActor treat the pass as best-effort-failed; it
|
||||
// logs a warning and retries reconcile on the next node startup.
|
||||
_log.Warning(
|
||||
"Cannot forward ReconcileSiteRequest for site {0} node {1} — no central ClusterClient registered",
|
||||
msg.SiteIdentifier, msg.NodeId);
|
||||
Sender.Tell(new Status.Failure(
|
||||
new InvalidOperationException("Central ClusterClient not registered")));
|
||||
return;
|
||||
}
|
||||
|
||||
_log.Debug(
|
||||
"Forwarding ReconcileSiteRequest for site {0} node {1} ({2} local instance(s)) to central",
|
||||
msg.SiteIdentifier, msg.NodeId, msg.LocalNameToRevisionHash.Count);
|
||||
_centralClient.Tell(
|
||||
new ClusterClient.Send("/user/central-communication", msg), Sender);
|
||||
});
|
||||
|
||||
// Internal: send heartbeat tick
|
||||
// Internal: send heartbeat tick.
|
||||
Receive<SendHeartbeat>(_ => SendHeartbeatToCentral());
|
||||
|
||||
// Internal: forward health report to central. The original Sender (the
|
||||
// AkkaHealthReportTransport's Ask) is forwarded as the ClusterClient.Send
|
||||
// sender so the central SiteHealthReportAck routes straight back to the
|
||||
// waiting Ask — making report delivery observable end-to-end (review 01
|
||||
// [Medium]). Mirrors the NotificationSubmit ack pattern above.
|
||||
Receive<SiteHealthReport>(msg =>
|
||||
{
|
||||
if (_centralClient == null)
|
||||
{
|
||||
// No ClusterClient registered yet. A non-accepted ack makes the
|
||||
// sender's counter-restore path treat this tick as a loss.
|
||||
_log.Warning(
|
||||
"Cannot forward SiteHealthReport #{0} — no central ClusterClient registered",
|
||||
msg.SequenceNumber);
|
||||
Sender.Tell(new SiteHealthReportAck(
|
||||
msg.SiteId, msg.SequenceNumber, Accepted: false,
|
||||
Error: "Central ClusterClient not registered"));
|
||||
return;
|
||||
}
|
||||
|
||||
_centralClient.Tell(
|
||||
new ClusterClient.Send("/user/central-communication", msg), Sender);
|
||||
});
|
||||
|
||||
// Internal: forward the periodic health report (health transport's Ask → ack back), so a
|
||||
// lost report is observable end-to-end and the sender can restore its per-interval counters.
|
||||
Receive<SiteHealthReport>(msg => _transport.ReportSiteHealth(msg, Sender));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -443,6 +334,12 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
|
||||
/// <inheritdoc />
|
||||
protected override void PreStart()
|
||||
{
|
||||
// Finalize the transport now that the actor context (and _log) exist. The default Akka
|
||||
// transport is given this actor's logging adapter so the "no ClusterClient registered"
|
||||
// warnings are preserved exactly. PreStart always runs before any message, so the Receive
|
||||
// closures above see a non-null _transport.
|
||||
_transport = _injectedTransport ?? new AkkaCentralTransport(_log);
|
||||
|
||||
_log.Info("SiteCommunicationActor started for site {0}", _siteId);
|
||||
|
||||
// Schedule periodic heartbeat to central. Uses the application heartbeat
|
||||
@@ -478,9 +375,6 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
|
||||
|
||||
private void SendHeartbeatToCentral()
|
||||
{
|
||||
if (_centralClient == null)
|
||||
return;
|
||||
|
||||
var hostname = Environment.MachineName;
|
||||
|
||||
// Stamp HeartbeatMessage.IsActive with this node's
|
||||
@@ -512,8 +406,17 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
|
||||
IsActive: isActive,
|
||||
DateTimeOffset.UtcNow);
|
||||
|
||||
_centralClient.Tell(
|
||||
new ClusterClient.Send("/user/central-communication", heartbeat), Self);
|
||||
// Fire-and-forget on both transports: a failure here must never fault the heartbeat timer
|
||||
// path. Both real transports swallow their own errors; this catch is a belt-and-braces
|
||||
// guarantee that no transport (including a future one) can turn a heartbeat into a fault.
|
||||
try
|
||||
{
|
||||
_transport.SendHeartbeat(heartbeat, Self);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Debug(ex, "Heartbeat send for site {0} failed; swallowed (heartbeats are fire-and-forget)", _siteId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
Reference in New Issue
Block a user