feat(siteruntime): per-node CertStore actor + trust broadcast to both site nodes (T17)

This commit is contained in:
Joseph Doherty
2026-06-18 03:13:48 -04:00
parent 303385fd98
commit c8d9303031
6 changed files with 526 additions and 0 deletions
@@ -1,4 +1,5 @@
using Akka.Actor;
using Akka.Cluster;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
@@ -167,6 +168,15 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
Receive<ReadTagValuesCommand>(msg =>
Context.ActorSelection("/user/dcl-manager").Tell(msg, Sender));
// T17 / D6 — OPC UA server-certificate trust. Trust is site-local: the
// trusted-peer PKI store is per-node, so a trust/remove MUST reach BOTH
// site nodes (node-a + node-b) or they diverge across failover. This
// singleton fans the corresponding per-node message out to the
// CertStoreActor on EVERY site node; list is answered from this node.
Receive<TrustServerCertCommand>(HandleTrustServerCert);
Receive<RemoveServerCertCommand>(HandleRemoveServerCert);
Receive<ListServerCertsCommand>(HandleListServerCerts);
// Internal startup messages
Receive<StartupConfigsLoaded>(HandleStartupConfigsLoaded);
Receive<SharedScriptsLoaded>(HandleSharedScriptsLoaded);
@@ -701,6 +711,120 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
}).PipeTo(sender);
}
// ── T17 / D6 — OPC UA server-certificate trust ──
/// <summary>
/// The base cluster role every site node carries (in addition to its
/// per-site role <c>site-{SiteId}</c>). Used to enumerate the site nodes a
/// trust/remove must reach so node-a and node-b PKI stores stay in lock-step
/// across failover (D6). Matches the role string set in <c>NodeOptions.Role</c>
/// for site hosts (see <c>AkkaHostedService.BuildRoles</c>).
/// </summary>
private const string SiteClusterRole = "Site";
/// <summary>
/// The per-node ask timeout for a cert broadcast. A node that does not ack
/// within this window is reported as a partial failure (the command is not
/// failed wholesale).
/// </summary>
private static readonly TimeSpan CertBroadcastTimeout = TimeSpan.FromSeconds(5);
/// <summary>
/// T17 / D6: broadcasts a trust to the per-node <see cref="CertStoreActor"/>
/// on every Up site node so both PKI stores receive the certificate.
/// </summary>
private void HandleTrustServerCert(TrustServerCertCommand command) =>
BroadcastToSiteCertStores(
new WriteCertToLocalStore(command.DerBase64, command.Thumbprint),
$"trust cert {command.Thumbprint} (connection {command.ConnectionName})");
/// <summary>
/// T17 / D6: broadcasts a remove to the per-node <see cref="CertStoreActor"/>
/// on every Up site node so the certificate leaves both PKI stores.
/// </summary>
private void HandleRemoveServerCert(RemoveServerCertCommand command) =>
BroadcastToSiteCertStores(
new RemoveCertFromLocalStore(command.Thumbprint),
$"remove cert {command.Thumbprint}");
/// <summary>
/// T17: lists this node's trusted + rejected PKI stores by asking the LOCAL
/// <see cref="CertStoreActor"/> (the singleton's own node). The list reflects
/// the active node's view; a trust broadcast keeps the standby in sync.
/// </summary>
private void HandleListServerCerts(ListServerCertsCommand command)
{
var sender = Sender;
var local = Context.ActorSelection($"/user/{CertStoreActor.WellKnownName}");
local.Ask<LocalCertOpAck>(new ListLocalCerts(), CertBroadcastTimeout)
.ContinueWith(t =>
{
if (t.IsCompletedSuccessfully)
{
var ack = t.Result;
return new CertTrustResult(ack.Success, ack.Error, ack.Certs);
}
var err = t.Exception?.GetBaseException().Message ?? "cert store did not respond";
_logger.LogWarning("Local cert store list failed: {Error}", err);
return new CertTrustResult(false, err, null);
}).PipeTo(sender);
}
/// <summary>
/// Fans <paramref name="localMessage"/> out to the <see cref="CertStoreActor"/>
/// on every Up site node, asks each with a short timeout, and aggregates the
/// acks into one <see cref="CertTrustResult"/>. A node that fails to ack is a
/// partial failure (Success=false, first error reported) — it never throws,
/// so a single unreachable standby cannot stall the singleton.
/// </summary>
private void BroadcastToSiteCertStores(object localMessage, string description)
{
var sender = Sender;
var cluster = Cluster.Get(Context.System);
var targets = cluster.State.Members
.Where(m => m.Status == MemberStatus.Up && m.HasRole(SiteClusterRole))
.Select(m => m.Address)
.ToList();
if (targets.Count == 0)
{
_logger.LogWarning("No Up site nodes found to {Description}; nothing trusted", description);
sender.Tell(new CertTrustResult(false, "no site nodes available", null));
return;
}
_logger.LogInformation("Broadcasting cert op to {Count} site node(s): {Description}",
targets.Count, description);
var asks = targets.Select(address =>
{
var path = new RootActorPath(address) / "user" / CertStoreActor.WellKnownName;
return Context.ActorSelection(path)
.Ask<LocalCertOpAck>(localMessage, CertBroadcastTimeout)
.ContinueWith(t => t.IsCompletedSuccessfully
? t.Result
: new LocalCertOpAck(false,
$"node {address} did not ack: {t.Exception?.GetBaseException().Message}",
null));
}).ToArray();
Task.WhenAll(asks).ContinueWith(t =>
{
// ContinueWith on WhenAll over per-task error-trapping completions
// never faults, so t.Result is always populated.
var acks = t.Result;
var allSucceeded = acks.All(a => a.Success);
var firstError = acks.FirstOrDefault(a => !a.Success)?.Error;
if (!allSucceeded)
{
_logger.LogWarning("Cert broadcast partial/total failure ({Description}): {Error}",
description, firstError);
}
return new CertTrustResult(allSucceeded, firstError, null);
}).PipeTo(sender);
}
// ── DCL connection management ──
/// <summary>