feat(site-runtime): cert-trust reconcile-on-join — singleton pushes trusted certs to (re)joining site nodes (UA1)

This commit is contained in:
Joseph Doherty
2026-07-09 01:22:13 -04:00
parent 39976772d9
commit c457e8f464
2 changed files with 215 additions and 0 deletions
@@ -270,6 +270,14 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
Receive<RemoveServerCertCommand>(HandleRemoveServerCert);
Receive<ListServerCertsCommand>(HandleListServerCerts);
// UA1: cert-trust reconcile-on-join. A (re)joining site node may have
// missed a trust broadcast while it was down, leaving its PKI store
// divergent forever. The singleton reads its own trusted store and pushes
// each cert to the joining node so the two per-node stores reconverge.
Receive<ClusterEvent.MemberUp>(HandleMemberUp);
Receive<SiteNodeJoined>(HandleSiteNodeJoined);
Receive<LocalCertExport>(HandleLocalCertExport);
// Internal startup messages
Receive<StartupConfigsLoaded>(HandleStartupConfigsLoaded);
// S5: re-attempt the startup deployed-config load after a transient
@@ -306,9 +314,28 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
_healthCollector?.SetActiveNode(true);
_logger.LogInformation("DeploymentManagerActor starting — loading deployed configs from SQLite...");
// UA1: watch for site nodes coming Up so we can reconcile cert trust onto
// them. Cluster-only concern — a no-op in the unit-test ActorSystem which
// has no cluster provider.
if (ClusterAvailable(Context.System))
{
Cluster.Get(Context.System).Subscribe(Self,
ClusterEvent.SubscriptionInitialStateMode.InitialStateAsEvents,
typeof(ClusterEvent.MemberUp));
}
LoadDeployedConfigs();
}
/// <summary>
/// True when this <see cref="ActorSystem"/> is running the cluster ref-provider.
/// The unit-test harness uses the local provider, where <see cref="Cluster.Get"/>
/// would throw — the cert reconcile-on-join subscription is skipped there.
/// </summary>
private static bool ClusterAvailable(ActorSystem system) =>
system is ExtendedActorSystem ext &&
ext.Provider.GetType().Name.Contains("Cluster", StringComparison.Ordinal);
/// <summary>
/// Loads all deployed configs off the actor thread and pipes the result back as a
/// <see cref="StartupConfigsLoaded"/>. Called from <see cref="PreStart"/> and re-invoked
@@ -330,6 +357,10 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
protected override void PostStop()
{
_healthCollector?.SetActiveNode(false);
if (ClusterAvailable(Context.System))
{
Cluster.Get(Context.System).Unsubscribe(Self);
}
base.PostStop();
}
@@ -1258,6 +1289,77 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
}).PipeTo(sender);
}
/// <summary>
/// The site node most recently observed coming Up, awaiting its trusted-cert
/// reconcile push. A site cluster is two nodes, so at most one peer reconciles
/// at a time; a single pending slot is sufficient (a second join before the
/// first export returns simply reconciles the newer node — both then converge).
/// </summary>
private Address? _pendingCertReconcileTarget;
/// <summary>
/// UA1: a site node came Up. On the singleton this is the trigger to reconcile
/// cert trust onto that node. Self and non-site members are ignored — self is
/// already the source of truth, and only site nodes host a CertStoreActor.
/// </summary>
private void HandleMemberUp(ClusterEvent.MemberUp evt)
{
var member = evt.Member;
if (!member.HasRole(SiteClusterRole))
return;
if (member.Address.Equals(Cluster.Get(Context.System).SelfAddress))
return;
Self.Tell(new SiteNodeJoined(member.Address));
}
/// <summary>
/// Reads this node's own trusted-peer store (via the LOCAL CertStoreActor) so
/// its contents can be pushed to the joined node. The export reply is handled
/// by <see cref="HandleLocalCertExport"/>, which uses the target captured here.
/// </summary>
private void HandleSiteNodeJoined(SiteNodeJoined msg)
{
_pendingCertReconcileTarget = msg.Address;
Context.ActorSelection($"/user/{CertStoreActor.WellKnownName}")
.Tell(new ExportLocalTrustedCerts());
}
/// <summary>
/// Pushes each locally-trusted certificate to the joined node's CertStoreActor.
/// Additive union only: removals are NOT reconciled here (there is no central
/// persistence of cert trust yet — a logged follow-up), and each write is
/// idempotent (overwrites by thumbprint), so re-pushing an already-trusted cert
/// is harmless. Fire-and-forget: a failed push heals on the next MemberUp or a
/// manual re-broadcast.
/// </summary>
private void HandleLocalCertExport(LocalCertExport export)
{
var target = _pendingCertReconcileTarget;
_pendingCertReconcileTarget = null;
if (target is null)
return;
if (!export.Success)
{
_logger.LogWarning(
"Cert-trust reconcile: could not read local trusted store for joining node {Address}: {Error}",
target, export.Error);
return;
}
var basePath = new RootActorPath(target) / "user" / CertStoreActor.WellKnownName;
foreach (var cert in export.Certs)
{
Context.ActorSelection(basePath).Tell(
new WriteCertToLocalStore(Convert.ToBase64String(cert.DerBytes), cert.Thumbprint));
}
_logger.LogInformation(
"Cert-trust reconcile: pushed {Count} trusted cert(s) to joining node {Address}",
export.Certs.Count, target);
}
// ── DCL connection management ──
/// <summary>
@@ -1928,6 +2030,14 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
// ── Internal messages ──
/// <summary>
/// UA1: a site cluster node has come Up and should receive a cert-trust
/// reconcile push. Derived from <see cref="ClusterEvent.MemberUp"/> (self and
/// non-site members already filtered out) so the reconcile flow is unit-testable
/// without constructing a real cluster membership event.
/// </summary>
internal sealed record SiteNodeJoined(Address Address);
internal record StartupConfigsLoaded(List<DeployedInstance> Configs, string? Error);
/// <summary>