Merge #867 Heidenhain integer leaves keep their declared type; #868 ConfigPublishCoordinator survives ConfigDb outage at start

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 12:29:00 -05:00
11 changed files with 720 additions and 41 deletions
@@ -52,8 +52,20 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
/// <c>ServerCluster.Enabled</c> is <b>not</b> consulted — nothing else in the codebase consults it,
/// so honouring it here would invent semantics. Disable the nodes, not the cluster row.
/// </para>
/// <para>
/// <b>Recovery load is asynchronous and retried (lmxopcua#868).</b> The in-flight recovery read used to run
/// synchronously in <c>PreStart</c>. When the ConfigDb was unreachable as the singleton started, the throw
/// became an <c>ActorInitializationException</c>, the default strategy STOPPED the actor, and the
/// <c>ClusterSingletonManager</c> never re-creates a singleton that terminated on its own — so every later
/// deployment was told to a dead singleton and hung. The read now runs off the actor thread, a failure
/// retries with exponential backoff (<see cref="DefaultLoadRetryMin"/> doubling to
/// <see cref="DefaultLoadRetryMax"/>, forever), and until it succeeds every message — deployments and acks
/// alike — is STASHED and replayed in order once the recovered state is in place, so a deploy arriving
/// during the outage is queued, never dropped. <see cref="ConfigPublishCoordinatorReadiness"/> reports the
/// loading state (and the last error) to the node's health endpoint as Degraded.
/// </para>
/// </summary>
public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers, IWithUnboundedStash
{
public const string DeploymentsTopic = "deployments";
public const string DeploymentAcksTopic = "deployment-acks";
@@ -61,8 +73,18 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
/// apply-outcome timeout is derived from.</summary>
public static readonly TimeSpan DefaultApplyDeadline = DeployTimings.ApplyDeadline;
/// <summary>First retry delay after a failed recovery load (lmxopcua#868).</summary>
public static readonly TimeSpan DefaultLoadRetryMin = TimeSpan.FromSeconds(1);
/// <summary>Ceiling of the recovery-load retry backoff (lmxopcua#868).</summary>
public static readonly TimeSpan DefaultLoadRetryMax = TimeSpan.FromSeconds(30);
private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
private readonly TimeSpan _applyDeadline;
private readonly TimeSpan _loadRetryMin;
private readonly TimeSpan _loadRetryMax;
private readonly ConfigPublishCoordinatorReadiness? _readiness;
private int _failedLoads;
/// <summary>
/// Where central→node commands are handed off (per-cluster mesh Phase 2), or
@@ -115,20 +137,30 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
/// <summary>Gets the timer scheduler for managing apply deadlines.</summary>
public ITimerScheduler Timers { get; set; } = null!;
/// <summary>Holds every message that arrives before the recovery load completes (lmxopcua#868).</summary>
public IStash Stash { get; set; } = null!;
/// <summary>Creates actor props for the ConfigPublishCoordinator with the specified configuration.</summary>
/// <param name="dbFactory">The database context factory for accessing configuration data.</param>
/// <param name="applyDeadline">The timeout for waiting for apply acknowledgments from cluster nodes.</param>
/// <returns>The Akka.NET <see cref="Akka.Actor.Props"/> used to spawn this actor.</returns>
/// <param name="meshRouter">
/// Central comm actor the dispatch is handed to, or <see langword="null"/> to publish on the
/// DistributedPubSub mediator directly (the original behaviour).
/// </param>
/// <param name="readiness">Where the recovery-load state is reported for health, or <see langword="null"/>.</param>
/// <param name="loadRetryMin">First recovery-load retry delay; <see langword="null"/> = <see cref="DefaultLoadRetryMin"/>.</param>
/// <param name="loadRetryMax">Recovery-load retry ceiling; <see langword="null"/> = <see cref="DefaultLoadRetryMax"/>.</param>
/// <returns>The Akka.NET <see cref="Akka.Actor.Props"/> used to spawn this actor.</returns>
public static Props Props(
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
TimeSpan? applyDeadline = null,
IActorRef? meshRouter = null) =>
Akka.Actor.Props.Create(() =>
new ConfigPublishCoordinator(dbFactory, applyDeadline ?? DefaultApplyDeadline, meshRouter));
IActorRef? meshRouter = null,
ConfigPublishCoordinatorReadiness? readiness = null,
TimeSpan? loadRetryMin = null,
TimeSpan? loadRetryMax = null) =>
Akka.Actor.Props.Create(() => new ConfigPublishCoordinator(
dbFactory, applyDeadline ?? DefaultApplyDeadline, meshRouter, readiness,
loadRetryMin ?? DefaultLoadRetryMin, loadRetryMax ?? DefaultLoadRetryMax));
/// <summary>
/// Initializes a new instance of the <see cref="ConfigPublishCoordinator"/> class.
@@ -136,45 +168,167 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
/// <param name="dbFactory">The database context factory for persisting deployment state.</param>
/// <param name="applyDeadline">The timeout for waiting for per-node apply acknowledgments.</param>
/// <param name="meshRouter">Central comm actor, or <see langword="null"/> for the mediator.</param>
/// <param name="readiness">Where the recovery-load state is reported, or <see langword="null"/>.</param>
/// <param name="loadRetryMin">First recovery-load retry delay.</param>
/// <param name="loadRetryMax">Recovery-load retry ceiling.</param>
public ConfigPublishCoordinator(
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
TimeSpan applyDeadline,
IActorRef? meshRouter = null)
IActorRef? meshRouter = null,
ConfigPublishCoordinatorReadiness? readiness = null,
TimeSpan? loadRetryMin = null,
TimeSpan? loadRetryMax = null)
{
_dbFactory = dbFactory;
_applyDeadline = applyDeadline;
_meshRouter = meshRouter;
_readiness = readiness;
_loadRetryMin = loadRetryMin ?? DefaultLoadRetryMin;
_loadRetryMax = loadRetryMax ?? DefaultLoadRetryMax;
Loading();
}
/// <summary>
/// The state before the recovery load has succeeded: only the load's own outcome and retry tick are
/// handled; everything else is stashed and replayed, in arrival order, once <see cref="Ready"/> is in force.
/// </summary>
private void Loading()
{
Receive<RecoveryLoaded>(HandleRecoveryLoaded);
Receive<RecoveryLoadFailed>(HandleRecoveryLoadFailed);
Receive<RetryRecoveryLoad>(_ => StartRecoveryLoad());
Receive<SubscribeAck>(_ => { /* DPS subscribe confirmation */ });
Receive<DispatchDeployment>(msg =>
{
_log.Info(
"DispatchDeployment {Id} queued: the coordinator has not yet loaded its recovery state from the " +
"ConfigDb ({Failed} failed attempt(s) so far); it is dispatched as soon as the load succeeds",
msg.DeploymentId, _failedLoads);
Stash.Stash();
});
ReceiveAny(_ => Stash.Stash());
}
/// <summary>The normal operating state, entered once the recovery load has been applied.</summary>
private void Ready()
{
Receive<DispatchDeployment>(HandleDispatch);
Receive<ApplyAck>(HandleAck);
Receive<DeadlineElapsed>(HandleDeadline);
Receive<SubscribeAck>(_ => { /* DPS subscribe confirmation */ });
// A late load outcome / retry tick cannot normally arrive once Ready; ignored, never stashed.
Receive<RecoveryLoaded>(_ => { });
Receive<RecoveryLoadFailed>(_ => { });
Receive<RetryRecoveryLoad>(_ => { });
}
/// <inheritdoc />
protected override void PreStart()
{
// On startup recover any deployment that was mid-flight when a prior singleton instance
// died. We re-derive _expectedAcks from NodeDeploymentState, replay the ACKs
// that already landed in the DB, and resume the deadline timer.
// Subscribe to per-node ApplyAck broadcasts so DriverHostActors on remote members can
// route their ACKs to whichever node currently hosts this singleton.
DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(DeploymentAcksTopic, Self));
using var db = _dbFactory.CreateDbContext();
// lmxopcua#868: NO database access here. A throw from PreStart stops the actor for good (the
// singleton manager does not re-create it), so the recovery read runs asynchronously and retries.
_readiness?.MarkLoading(_failedLoads, lastError: null);
StartRecoveryLoad();
}
/// <inheritdoc />
protected override void PostStop()
{
// A restart re-enters PreStart (Loading again); a stop or hand-over leaves this node not hosting it.
_readiness?.MarkNotHosted();
base.PostStop();
}
/// <summary>
/// Reads the deployment that was mid-flight when a prior singleton instance died, off the actor thread,
/// and pipes the outcome back. Touches no actor state.
/// </summary>
private void StartRecoveryLoad()
{
LoadRecoveryAsync(_dbFactory).PipeTo(
Self,
success: loaded => loaded,
failure: ex => new RecoveryLoadFailed(ex is AggregateException { InnerException: { } inner } ? inner : ex));
}
private static async Task<RecoveryLoaded> LoadRecoveryAsync(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory)
{
// Yield first so a factory or provider that throws synchronously still faults the TASK (piped back as
// RecoveryLoadFailed) rather than the caller on the actor thread.
await Task.Yield();
await using var db = await dbFactory.CreateDbContextAsync().ConfigureAwait(false);
// Projected: the row carries the whole ArtifactBlob (varbinary(max)), which this recovery never
// reads — and this query runs on every singleton start and hand-over.
var inflight = db.Deployments
var inflight = await db.Deployments
.Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks)
.OrderByDescending(d => d.CreatedAtUtc)
.Select(d => new { d.DeploymentId, d.RevisionHash, d.CreatedAtUtc })
.FirstOrDefault();
.FirstOrDefaultAsync()
.ConfigureAwait(false);
if (inflight is null) return new RecoveryLoaded(null);
var nodeStates = await db.NodeDeploymentStates
.Where(x => x.DeploymentId == inflight.DeploymentId)
.AsNoTracking()
.Select(x => new RecoveredNodeState(x.NodeId, x.Status, x.FailureReason))
.ToListAsync()
.ConfigureAwait(false);
return new RecoveryLoaded(new RecoveredDeployment(
inflight.DeploymentId, inflight.RevisionHash, inflight.CreatedAtUtc, nodeStates));
}
private void HandleRecoveryLoadFailed(RecoveryLoadFailed msg)
{
_failedLoads++;
// Exponential backoff, capped: min * 2^(n-1), with the shift bounded so it never overflows.
var factor = 1L << Math.Min(_failedLoads - 1, 20);
var delay = TimeSpan.FromTicks(Math.Min(_loadRetryMax.Ticks, _loadRetryMin.Ticks * factor));
_readiness?.MarkLoading(_failedLoads, msg.Error.Message);
_log.Warning(msg.Error,
"ConfigPublishCoordinator could not load its recovery state from the ConfigDb (attempt {Attempt}); " +
"retrying in {Delay}. Deployments and acks are queued until it succeeds",
_failedLoads, delay);
Timers.StartSingleTimer(LoadRetryTimerKey, RetryRecoveryLoad.Instance, delay);
}
/// <summary>
/// Applies the recovered state, switches to <see cref="Ready"/> and replays every stashed message. The
/// recovery DECISIONS (re-dispatch, time out) run inline, before the replay, so a queued message for the
/// same deployment is judged against the recovered state rather than racing a self-sent one.
/// </summary>
private void HandleRecoveryLoaded(RecoveryLoaded msg)
{
Timers.Cancel(LoadRetryTimerKey);
if (_failedLoads > 0)
_log.Info("ConfigPublishCoordinator loaded its recovery state after {Failed} failed attempt(s)", _failedLoads);
_failedLoads = 0;
Become(Ready);
_readiness?.MarkReady();
try
{
ApplyRecovery(msg.Inflight);
}
finally
{
Stash.UnstashAll();
}
}
/// <summary>
/// Recovers the deployment that was mid-flight when a prior singleton instance died: re-derives
/// <c>_expectedAcks</c> from <c>NodeDeploymentState</c>, replays the acks that already landed, and resumes
/// the deadline timer.
/// </summary>
private void ApplyRecovery(RecoveredDeployment? inflight)
{
if (inflight is null) return;
_current = new DeploymentId(inflight.DeploymentId);
var nodeStates = db.NodeDeploymentStates
.Where(x => x.DeploymentId == inflight.DeploymentId)
.AsNoTracking()
.ToList();
var nodeStates = inflight.NodeStates;
// Resume the deadline timer using the remaining time. The deadline runs from when the
// deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough
@@ -201,12 +355,12 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
"before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " +
"re-dispatching with {Remaining} of the apply deadline left",
_current, remaining);
Self.Tell(new DispatchDeployment(
HandleDispatch(new DispatchDeployment(
_current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId()));
}
else
{
Self.Tell(new DeadlineElapsed(_current.Value));
HandleDeadline(new DeadlineElapsed(_current.Value));
}
return;
}
@@ -222,18 +376,25 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
if (remaining <= TimeSpan.Zero)
{
Self.Tell(new DeadlineElapsed(_current.Value));
}
else
{
Timers.StartSingleTimer(DeadlineTimerKey, new DeadlineElapsed(_current.Value), remaining);
HandleDeadline(new DeadlineElapsed(_current.Value));
return;
}
Timers.StartSingleTimer(DeadlineTimerKey, new DeadlineElapsed(_current.Value), remaining);
_log.Info("Coordinator recovered in-flight deployment {Id} ({Acked}/{Total} acks landed)",
_current, _acks.Count, _expectedAcks.Count);
}
private void HandleDispatch(DispatchDeployment msg)
{
// A duplicate of the deployment already in flight with a seeded expected set (a dispatch queued while the
// recovery load ran, for the very row that load re-drove) must not reset the acks already gathered.
if (_current == msg.DeploymentId && _expectedAcks.Count > 0)
{
_log.Debug("Ignoring duplicate DispatchDeployment {Id}: already in flight", msg.DeploymentId);
return;
}
// Terminal-state guard (fix round 1): a late or re-delivered dispatch for a row that is already
// Sealed / PartiallyFailed / TimedOut — e.g. one AdminOps closed as a lost dispatch while this message
// was still queued — must never flip it back to AwaitingApplyAcks or seed an expected set for it.
@@ -519,9 +680,27 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers
}
private const string DeadlineTimerKey = "apply-deadline";
private const string LoadRetryTimerKey = "recovery-load-retry";
public sealed record DeadlineElapsed(DeploymentId DeploymentId);
/// <summary>The recovery read's result: the newest in-flight deployment, or null when none.</summary>
private sealed record RecoveryLoaded(RecoveredDeployment? Inflight);
/// <summary>The recovery read failed (ConfigDb unreachable, ...); retried with backoff.</summary>
private sealed record RecoveryLoadFailed(Exception Error);
/// <summary>The backoff timer's tick: run the recovery read again.</summary>
private sealed class RetryRecoveryLoad
{
public static readonly RetryRecoveryLoad Instance = new();
}
private sealed record RecoveredDeployment(
Guid DeploymentId, string RevisionHash, DateTime CreatedAtUtc, IReadOnlyList<RecoveredNodeState> NodeStates);
private sealed record RecoveredNodeState(string NodeId, NodeDeploymentStatus Status, string? FailureReason);
/// <summary>
/// Stages a status change WITHOUT materialising the row: the row carries the whole <c>ArtifactBlob</c>
/// (varbinary(max), multi-MB on a real fleet), which no status transition reads (fix round 1). A stub
@@ -0,0 +1,50 @@
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
/// <summary>Where the <see cref="ConfigPublishCoordinator"/> singleton stands on this node (lmxopcua#868).</summary>
public enum ConfigPublishCoordinatorState
{
/// <summary>The singleton is not running on this node (the partner hosts it, or it has not started yet).</summary>
NotHosted,
/// <summary>Running here, but its recovery state has not loaded from the ConfigDb yet; deployments are queued.</summary>
Loading,
/// <summary>Running here with its recovery state loaded; deployments are dispatched.</summary>
Ready,
}
/// <summary>One immutable observation of the coordinator's state.</summary>
/// <param name="State">The state.</param>
/// <param name="FailedLoadAttempts">Consecutive failed recovery loads (0 unless <see cref="ConfigPublishCoordinatorState.Loading"/>).</param>
/// <param name="LastError">The last load failure's message, or null.</param>
/// <param name="SinceUtc">When the state was entered.</param>
public sealed record ConfigPublishCoordinatorSnapshot(
ConfigPublishCoordinatorState State, int FailedLoadAttempts, string? LastError, DateTime SinceUtc);
/// <summary>
/// Thread-safe state bridge from the <see cref="ConfigPublishCoordinator"/> actor to the node's health endpoint
/// (one DI singleton per admin node). The actor writes; the health check reads.
/// </summary>
public sealed class ConfigPublishCoordinatorReadiness
{
private volatile ConfigPublishCoordinatorSnapshot _current =
new(ConfigPublishCoordinatorState.NotHosted, 0, null, DateTime.UtcNow);
/// <summary>The latest observation.</summary>
public ConfigPublishCoordinatorSnapshot Current => _current;
/// <summary>The coordinator is loading (or retrying the load of) its recovery state.</summary>
/// <param name="failedAttempts">Failed attempts so far.</param>
/// <param name="lastError">The last failure's message, or null.</param>
public void MarkLoading(int failedAttempts, string? lastError)
{
var since = _current.State == ConfigPublishCoordinatorState.Loading ? _current.SinceUtc : DateTime.UtcNow;
_current = new(ConfigPublishCoordinatorState.Loading, failedAttempts, lastError, since);
}
/// <summary>The recovery state is loaded.</summary>
public void MarkReady() => _current = new(ConfigPublishCoordinatorState.Ready, 0, null, DateTime.UtcNow);
/// <summary>The singleton stopped on this node.</summary>
public void MarkNotHosted() => _current = new(ConfigPublishCoordinatorState.NotHosted, 0, null, DateTime.UtcNow);
}
@@ -112,7 +112,11 @@ public static class ServiceCollectionExtensions
// coordinator. Deliberately NOT an ActorSelection.ResolveOne here: that blocks
// inside a props factory and races the spawn it is waiting for.
var meshRouter = registry.Get<CentralCommunicationActorKey>();
return ConfigPublishCoordinator.Props(dbFactory, applyDeadline: DeployTimings.ApplyDeadline, meshRouter: meshRouter);
// lmxopcua#868: optional — through IServiceProvider for GetService's null-for-absent contract
// (the Akka resolver makes no such promise), so harnesses that never register it still boot.
var readiness = resolver.GetService<IServiceProvider>()?.GetService<ConfigPublishCoordinatorReadiness>();
return ConfigPublishCoordinator.Props(
dbFactory, applyDeadline: DeployTimings.ApplyDeadline, meshRouter: meshRouter, readiness: readiness);
},
singletonOptions);
@@ -0,0 +1,45 @@
using Microsoft.Extensions.Diagnostics.HealthChecks;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
namespace ZB.MOM.WW.OtOpcUa.Host.Health;
/// <summary>
/// The deploy coordinator's readiness (lmxopcua#868): <b>Degraded</b> while the
/// <see cref="ConfigPublishCoordinator"/> singleton on this node has not yet loaded its recovery state from the
/// ConfigDb (deployments are queued, not dispatched), Healthy otherwise — including on the admin node that does
/// not host the singleton. Never Unhealthy: the node still serves (the configdb probe owns "DB down").
/// </summary>
/// <param name="readiness">The coordinator's state bridge; null on graphs that never register one.</param>
public sealed class ConfigPublishCoordinatorHealthCheck(ConfigPublishCoordinatorReadiness? readiness) : IHealthCheck
{
/// <summary>The check's registration name.</summary>
public const string Name = "config-publish";
/// <inheritdoc />
public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
{
if (readiness is null)
return Task.FromResult(HealthCheckResult.Healthy("Deploy coordinator readiness is not tracked on this node."));
var s = readiness.Current;
var data = new Dictionary<string, object>(StringComparer.Ordinal)
{
["state"] = s.State.ToString(),
["sinceUtc"] = s.SinceUtc.ToString("O"),
["failedLoadAttempts"] = s.FailedLoadAttempts,
};
if (s.LastError is not null) data["lastError"] = s.LastError;
return Task.FromResult(s.State switch
{
ConfigPublishCoordinatorState.Loading => HealthCheckResult.Degraded(
s.FailedLoadAttempts == 0
? "Deploy coordinator is loading its recovery state from the ConfigDb; deployments are queued until it completes."
: $"Deploy coordinator cannot load its recovery state from the ConfigDb ({s.FailedLoadAttempts} failed attempt(s), " +
$"last: {s.LastError}); retrying with backoff. Deployments are queued until it succeeds.",
data: data),
ConfigPublishCoordinatorState.Ready => HealthCheckResult.Healthy("Deploy coordinator ready.", data),
_ => HealthCheckResult.Healthy("Deploy coordinator singleton is not hosted on this node.", data),
});
}
}
@@ -87,6 +87,15 @@ public static class HealthEndpoints
{
ProbeQuery = static (db, ct) => db.Deployments.AsNoTracking().Take(1).ToListAsync(ct),
});
// lmxopcua#868: Degraded (still 200) while the deploy coordinator singleton on this node is retrying
// its ConfigDb recovery load and queueing deployments; Healthy otherwise. Ready tier only.
checks.Add(new HealthCheckRegistration(
ConfigPublishCoordinatorHealthCheck.Name,
sp => new ConfigPublishCoordinatorHealthCheck(
sp.GetService<ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators.ConfigPublishCoordinatorReadiness>()),
failureStatus: null,
tags: new[] { ZbHealthTags.Ready }));
}
checks
@@ -176,7 +176,12 @@ else
builder.Services.Configure<AclOptions>(builder.Configuration.GetSection(AclOptions.SectionName));
if (hasAdmin)
{
builder.Services.AddOtOpcUaConfigDb(builder.Configuration);
// lmxopcua#868: the deploy coordinator singleton reports its ConfigDb recovery-load state here; the
// config-publish health check (AddOtOpcUaHealth) reads it — Degraded while deployments are queued.
builder.Services.AddSingleton<ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators.ConfigPublishCoordinatorReadiness>();
}
else
Log.Information("Driver-only node — no ConfigDb registered; configuration via ConfigSource:Mode=FetchAndCache");