dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(reconcile): batch startup reconcile into fixed round trips, bound central work by QueryTimeout, retry failed site passes with back-off (review 2026-10-21 03/P-1)

Browse Source
This commit is contained in:
Joseph Doherty
2026-10-22 06:25:57 -04:00
parent d6a9a79c27
commit 7c83a4aa94
9 changed files with 638 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/SiteReconciliationActor.cs
+80 -18
View File
@@ -8,8 +8,8 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
/// <summary>
/// Runs on EVERY site node (NOT a singleton) so a standby that was DOWN during a deploy
/// self-heals on its next restart. On startup the actor performs one best-effort
/// reconciliation pass:
/// self-heals on its next restart. On startup the actor performs a best-effort
/// reconciliation pass (retried a bounded number of times, see remarks):
/// <list type="number">
/// <item>read the node's local deployed inventory from SQLite,</item>
/// <item>report it to central via the <c>SiteCommunicationActor</c> Ask
@@ -23,10 +23,18 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
/// <remarks>
/// <para>
/// <b>Best-effort throughout.</b> A central-unreachable / timed-out Ask is caught, logged
/// at Warning, and the pass simply ends — reconcile re-runs on the next node startup; it is a
/// self-heal, not a critical path. A per-item fetch/write failure is caught and logged, then
/// the remaining gap items continue (one bad item must not abort the rest). The actor never
/// crashes on these failures.
/// at Warning, and the pass ends; it is a self-heal, not a critical path. A per-item
/// fetch/write failure is caught and logged, then the remaining gap items continue (one bad
/// item must not abort the rest). The actor never crashes on these failures.
/// </para>
/// <para>
/// <b>Bounded retry.</b> A pass that did not complete (central unreachable, Ask timeout,
/// local read error) or completed with failed items is re-run, up to <c>maxAttempts</c>
/// passes in total, after an exponential back-off (<c>retryBaseDelay</c>, then 2x, 4x...).
/// Each retry re-reads the local inventory, so items an earlier pass healed are no longer in
/// the gap. Before this, a failed pass waited for the NEXT node restart — leaving exactly the
/// node this actor exists to heal unhealed. After the last attempt the actor logs and stops
/// trying until the next startup.
/// </para>
/// <para>
/// The pass runs after a small startup delay (so the site's central transport has time to
@@ -63,6 +71,12 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
private readonly ILogger<SiteReconciliationActor> _logger;
private readonly TimeSpan _initialDelay;
private readonly TimeSpan _askTimeout;
private readonly int _maxAttempts;
private readonly TimeSpan _retryBaseDelay;
// Number of passes started so far (1-based once the first pass runs). Only touched on
// the actor thread.
private int _attempt;
/// <summary>Akka timer scheduler injected by the framework via <see cref="IWithTimers"/>.</summary>
public ITimerScheduler Timers { get; set; } = null!;
@@ -93,6 +107,13 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
/// node's reconcile pass found the gap. Optional: when null the pass stores configs and
/// starts nothing, exactly as it did before.
/// </param>
/// <param name="maxAttempts">
/// Total passes allowed per actor lifetime when a pass fails or leaves failed items.
/// Defaults to 3; values below 1 are treated as 1 (no retry).
/// </param>
/// <param name="retryBaseDelay">
/// Back-off before the first retry, doubled for each later one. Defaults to 30 seconds.
/// </param>
public SiteReconciliationActor(
SiteStorageService storage,
IDeploymentConfigFetcher configFetcher,
@@ -102,7 +123,9 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
ILogger<SiteReconciliationActor> logger,
TimeSpan? initialDelay = null,
TimeSpan? askTimeout = null,
IActorRef? deploymentManager = null)
IActorRef? deploymentManager = null,
int maxAttempts = 3,
TimeSpan? retryBaseDelay = null)
{
_storage = storage;
_configFetcher = configFetcher;
@@ -113,23 +136,33 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
_logger = logger;
_initialDelay = initialDelay ?? TimeSpan.FromSeconds(5);
_askTimeout = askTimeout ?? TimeSpan.FromSeconds(30);
_maxAttempts = Math.Max(1, maxAttempts);
_retryBaseDelay = retryBaseDelay ?? TimeSpan.FromSeconds(30);
Receive<RunReconcile>(_ => RunReconcilePassAsync().PipeTo(Self));
Receive<RunReconcile>(_ =>
{
_attempt++;
RunReconcilePassAsync().PipeTo(Self);
});
Receive<ReconcilePassResult>(HandleReconcilePassResult);
// Defensive: RunReconcilePassAsync is designed never to throw (it returns a faulted
// ReconcilePassResult instead), but if anything unexpected faults the piped Task the
// Status.Failure would otherwise go to dead letters silently. Log it instead.
Receive<Status.Failure>(f => _logger.LogWarning(f.Cause,
"Reconcile pass faulted unexpectedly for site {Site} node {Node}",
_siteIdentifier, _nodeId));
Receive<Status.Failure>(f =>
{
_logger.LogWarning(f.Cause,
"Reconcile pass faulted unexpectedly for site {Site} node {Node}",
_siteIdentifier, _nodeId);
ScheduleRetryIfAllowed();
});
}
/// <inheritdoc />
protected override void PreStart()
{
base.PreStart();
// One-shot pass after a small delay so the site's central transport can connect first.
// First pass after a small delay so the site's central transport can connect first.
// Non-blocking: the timer fires RunReconcile back onto this actor's mailbox.
Timers.StartSingleTimer(StartupTimerKey, RunReconcile.Instance, _initialDelay);
_logger.LogInformation(
@@ -239,16 +272,24 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
if (result.Error != null)
{
// Best-effort: a failed pass (central unreachable, Ask timeout, local read error) is
// logged at Warning and the actor stays alive. Reconcile re-runs on the next startup.
// logged at Warning and the actor stays alive; it is retried with back-off up to the
// attempt cap, then left to the next startup.
_logger.LogWarning(result.Error,
"Reconcile pass for site {Site} node {Node} did not complete (central unreachable or read error) — will retry on next startup",
_siteIdentifier, _nodeId);
"Reconcile pass {Attempt}/{MaxAttempts} for site {Site} node {Node} did not complete (central unreachable or read error)",
_attempt, _maxAttempts, _siteIdentifier, _nodeId);
ScheduleRetryIfAllowed();
return;
}
_logger.LogInformation(
"Reconcile pass for site {Site} node {Node} complete: {Fetched} fetched, {Failed} failed, {Orphans} orphan(s)",
_siteIdentifier, _nodeId, result.Fetched, result.Failed, result.Orphans);
"Reconcile pass {Attempt}/{MaxAttempts} for site {Site} node {Node} complete: {Fetched} fetched, {Failed} failed, {Orphans} orphan(s)",
_attempt, _maxAttempts, _siteIdentifier, _nodeId, result.Fetched, result.Failed, result.Orphans);
// Items that failed to fetch/store are still missing: retry the pass (the re-read
// inventory makes the gap only what is still missing). The healed instances below are
// started now regardless — a partial pass still healed them.
if (result.Failed > 0)
ScheduleRetryIfAllowed();
// Heal the RUNTIME, not just the file (row 67). Storing a config starts nothing: the
// Deployment Manager's boot-time SQLite load has already run by now, so without this
@@ -268,9 +309,30 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
}
}
/// <summary>
/// Schedules the next reconcile pass after an exponential back-off, unless the attempt cap
/// is reached — in which case the actor logs and leaves the heal to the next node startup.
/// </summary>
private void ScheduleRetryIfAllowed()
{
if (_attempt >= _maxAttempts)
{
_logger.LogWarning(
"Reconcile for site {Site} node {Node} gave up after {Attempts} attempt(s) — will retry on next startup",
_siteIdentifier, _nodeId, _attempt);
return;
}
var delay = TimeSpan.FromTicks(_retryBaseDelay.Ticks * (1L << Math.Min(_attempt - 1, 16)));
_logger.LogInformation(
"Reconcile for site {Site} node {Node}: retrying in {Delay} (attempt {Next}/{MaxAttempts})",
_siteIdentifier, _nodeId, delay, _attempt + 1, _maxAttempts);
Timers.StartSingleTimer(StartupTimerKey, RunReconcile.Instance, delay);
}
// ── Internal messages ──
/// <summary>Self-tick that drives the one-shot startup reconcile pass.</summary>
/// <summary>Self-tick that drives a startup reconcile pass (the first, or a retry).</summary>
private sealed class RunReconcile
{
public static readonly RunReconcile Instance = new();