fix(reconcile): batch startup reconcile into fixed round trips, bound central work by QueryTimeout, retry failed site passes with back-off (review 2026-10-21 03/P-1)

2026-10-22 06:25:57 -04:00
parent d6a9a79c27
commit 7c83a4aa94
9 changed files with 638 additions and 130 deletions
@@ -341,6 +341,37 @@ public interface IDeploymentManagerRepository
        string configurationJson, string token, DateTimeOffset createdAtUtc, DateTimeOffset expiresAtUtc,
        CancellationToken cancellationToken = default);

+    /// <summary>
+    /// Reads the deployed config snapshots of <paramref name="instanceIds"/> in ONE untracked
+    /// query — the startup-reconcile gap read. Only the gap's ids should be passed: this is the
+    /// read that materialises each snapshot's <c>ConfigurationJson</c>.
+    /// </summary>
+    /// <param name="instanceIds">Instances whose snapshots to read; duplicates are harmless.</param>
+    /// <param name="cancellationToken">A cancellation token that can be used to cancel the operation.</param>
+    /// <returns>Instance id → snapshot, omitting instances with no snapshot (deleted race).</returns>
+    Task<IReadOnlyDictionary<int, DeployedConfigSnapshot>> GetDeployedSnapshotsByInstanceIdsAsync(
+        IReadOnlyCollection<int> instanceIds, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Batch form of <see cref="StagePendingIfAbsentAsync"/> for startup reconcile: expired
+    /// pending rows for the candidates' instances are deleted, then every candidate whose
+    /// instance has no LIVE pending row is inserted, all in one <c>SaveChanges</c>. A live row
+    /// (an in-flight deploy, or the peer node's concurrent reconcile) is never superseded.
+    /// Self-contained: commits its own writes.
+    /// </summary>
+    /// <param name="candidates">One new pending row per instance (later duplicates for the same instance are ignored).</param>
+    /// <param name="nowUtc">"Now": rows with <c>ExpiresAtUtc &lt;= nowUtc</c> are expired.</param>
+    /// <param name="cancellationToken">A cancellation token that can be used to cancel the operation.</param>
+    /// <returns>
+    /// Instance id → the instance's live pending row after the call: the candidate itself
+    /// (reference-equal) when it was staged, otherwise the pre-existing live row, returned
+    /// WITHOUT its <c>ConfigurationJson</c> (empty string). An instance whose candidate could not
+    /// be staged and has no live row (e.g. the instance was deleted mid-reconcile) is omitted.
+    /// </returns>
+    Task<IReadOnlyDictionary<int, PendingDeployment>> StagePendingBatchIfAbsentAsync(
+        IReadOnlyList<PendingDeployment> candidates, DateTimeOffset nowUtc,
+        CancellationToken cancellationToken = default);
+
    // Instance lookups for deployment pipeline
    /// <summary>
    /// Gets an instance by ID.
@@ -1,5 +1,6 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
 using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
 using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
 using ZB.MOM.WW.ScadaBridge.Commons.Types.Deployment;
@@ -18,6 +19,11 @@ namespace ZB.MOM.WW.ScadaBridge.Communication;
 /// replicate. The node fetches each gap item's config over the existing token-gated HTTP
 /// endpoint; this service stages those tokens as <c>PendingDeployment</c> rows.
 ///
+/// The whole pass is a fixed number of round trips regardless of gap size: the expected-set
+/// projection, ONE untracked read of the gap's snapshot configs, and ONE batched
+/// insert-if-absent of the pending rows. Work is bounded by
+/// <see cref="CommunicationOptions.QueryTimeout"/> — the site's own deadline for the reply.
+///
 /// Scoped service (holds scoped repositories); resolved per-request by the actor inside a
 /// DI scope. The actor stays thin: it captures <c>Sender</c>, resolves this service in a
 /// scope, awaits <see cref="ReconcileAsync"/>, and pipes the response back.
@@ -73,92 +79,96 @@ public class ReconcileService
                Array.Empty<ReconcileGapItem>(), Array.Empty<string>(), baseUrl);
        }

-        // 2. Central's expected deployed set for this site (instances with a snapshot).
+        // Central does no more work than the site will wait for: the node's own Ask/gRPC
+        // deadline is QueryTimeout, so past it every staged token would go to a caller that
+        // has already given up. (The actor-lifetime token passed in is not request-scoped.)
+        using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+        if (_options.QueryTimeout > TimeSpan.Zero)
+            deadline.CancelAfter(_options.QueryTimeout);
+        var ct = deadline.Token;
+
+        // 2. Central's expected deployed set for this site (instances with a snapshot) — a
+        //    projection carrying each snapshot's DeploymentId/RevisionHash, no config JSON.
        var expected = await _deploymentRepository
-            .GetExpectedDeploymentsForSiteAsync(site.Id, cancellationToken)
+            .GetExpectedDeploymentsForSiteAsync(site.Id, ct)
            .ConfigureAwait(false);

        var localMap = request.LocalNameToRevisionHash;
-        var gap = new List<ReconcileGapItem>();
-
-        var now = DateTimeOffset.UtcNow;
-        var expiresAt = now + _options.PendingDeploymentTtl;

        // 3. GAP = expected items the node is MISSING (name absent locally) or STALE
        //    (local revision != expected revision). Current items are omitted.
-        foreach (var exp in expected)
-        {
-            var present = localMap.TryGetValue(exp.InstanceUniqueName, out var localHash);
-            var stale = present && !string.Equals(localHash, exp.RevisionHash, StringComparison.Ordinal);
-            if (present && !stale)
-                continue; // node already has the current revision
+        var gapExpected = expected
+            .Where(exp => !localMap.TryGetValue(exp.InstanceUniqueName, out var localHash)
+                          || !string.Equals(localHash, exp.RevisionHash, StringComparison.Ordinal))
+            .ToList();

-            // 4. Read the frozen snapshot config to stage. Null = the snapshot was deleted
-            //    between the expected-set query and now (instance removed mid-reconcile);
-            //    skip it — best-effort reconcile re-runs on the next node startup.
-            var snapshot = await _deploymentRepository
-                .GetDeployedSnapshotByInstanceIdAsync(exp.InstanceId, cancellationToken)
+        var gap = new List<ReconcileGapItem>(gapExpected.Count);
+        if (gapExpected.Count > 0)
+        {
+            // 4. ONE read of the frozen snapshot configs, for the gap only — this is the only
+            //    query that materialises ConfigurationJson. A missing entry means the snapshot
+            //    was deleted between the expected-set query and now (instance removed
+            //    mid-reconcile); it is skipped.
+            var snapshots = await _deploymentRepository
+                .GetDeployedSnapshotsByInstanceIdsAsync(
+                    gapExpected.Select(e => e.InstanceId).ToList(), ct)
                .ConfigureAwait(false);
-            if (snapshot == null)
+
+            var now = DateTimeOffset.UtcNow;
+            var expiresAt = now + _options.PendingDeploymentTtl;
+
+            // Stage with the snapshot's DeploymentId so the gap item's DeploymentId + token point
+            // the node at the right pending row to fetch.
+            var staging = new List<(ExpectedDeployment Expected, PendingDeployment Candidate)>(gapExpected.Count);
+            foreach (var exp in gapExpected)
            {
-                _logger.LogDebug(
-                    "Reconcile: snapshot for instance {Instance} disappeared (deleted race); skipping",
-                    exp.InstanceUniqueName);
-                continue;
+                if (!snapshots.TryGetValue(exp.InstanceId, out var snapshot))
+                {
+                    _logger.LogDebug(
+                        "Reconcile: snapshot for instance {Instance} disappeared (deleted race); skipping",
+                        exp.InstanceUniqueName);
+                    continue;
+                }
+
+                staging.Add((exp, new PendingDeployment(
+                    snapshot.DeploymentId, exp.InstanceId, snapshot.RevisionHash,
+                    snapshot.ConfigurationJson, DeploymentFetchToken.Generate(), now, expiresAt)));
            }

-            var token = DeploymentFetchToken.Generate();
-
-            // Stage with the snapshot's DeploymentId as the deploymentId so the gap item's
-            // DeploymentId + token point the node at the right pending row to fetch.
-            //
-            // StagePendingIfAbsent is insert-if-absent: if BOTH site nodes are concurrently
-            // missing the same instance (e.g. fresh container start / cleared SQLite after a
-            // successful deploy), both attempt to stage here. The first succeeds (true); the
-            // second gets false and is handled in the !staged branch below — it returns the
-            // existing pending row's token so it heals in the same round, rather than being
-            // omitted. Deploy-time supersession serializes via the per-instance operation lock.
-            var staged = await _deploymentRepository.StagePendingIfAbsentAsync(
-                exp.InstanceId, snapshot.DeploymentId, exp.RevisionHash,
-                snapshot.ConfigurationJson, token, now, expiresAt, cancellationToken)
+            // Insert-if-absent, in one save. If BOTH site nodes are concurrently missing the same
+            // instance (fresh container start / cleared SQLite after a successful deploy), both
+            // reconcile here; the loser finds the winner's LIVE row and is handed ITS
+            // DeploymentId/RevisionHash/Token (the fetch token is multi-use within its TTL), so
+            // both nodes heal in the same round instead of the second waiting for a restart. The
+            // same applies to an in-flight deploy's row: its config is newer than the snapshot,
+            // and fetching it is still correct (the site's guarded write handles ordering).
+            // Deploy-time supersession serializes via the per-instance operation lock.
+            var liveRows = await _deploymentRepository
+                .StagePendingBatchIfAbsentAsync(staging.Select(x => x.Candidate).ToList(), now, ct)
                .ConfigureAwait(false);

-            if (!staged)
+            foreach (var (exp, candidate) in staging)
            {
-                // A pending row already exists for this instance — either a CONCURRENT reconcile
-                // from the other site node (both nodes' SQLite empty after a fresh/cleared deploy,
-                // both reconciling at startup) or an in-flight deploy. Do NOT omit the item: if we
-                // did, the second concurrently-missing node would get 0 fetched and stay unhealed
-                // until a later restart. Instead, read the EXISTING pending row and emit a gap item
-                // carrying ITS DeploymentId/RevisionHash/Token. The fetch token is multi-use within
-                // its TTL, so both nodes fetch the same pending config and heal in the same round.
-                // (If the existing row is from an in-flight deploy its config is newer than the
-                // snapshot — fetching it is still correct; the site's guarded write handles ordering.)
-                var existing = await _deploymentRepository
-                    .GetPendingDeploymentByInstanceIdAsync(exp.InstanceId, now, cancellationToken)
-                    .ConfigureAwait(false);
-                if (existing != null)
+                if (!liveRows.TryGetValue(candidate.InstanceId, out var row))
+                {
+                    // Raced away: no live row could be staged or found (e.g. the instance was
+                    // deleted mid-reconcile). Omit it — the node retries on its next pass.
+                    _logger.LogDebug(
+                        "Reconcile: no pending row could be staged for instance {Instance}; omitting from gap",
+                        exp.InstanceUniqueName);
+                    continue;
+                }
+
+                if (!ReferenceEquals(row, candidate))
                {
                    _logger.LogDebug(
                        "Reconcile: pending row already exists for instance {Instance} (concurrent reconcile or in-flight deploy); returning existing token so this node heals too",
                        exp.InstanceUniqueName);
-                    gap.Add(new ReconcileGapItem(
-                        exp.InstanceUniqueName, existing.DeploymentId, existing.RevisionHash,
-                        exp.IsEnabled, existing.Token));
                }
-                else
-                {
-                    // Raced away: the pending row was purged between the stage attempt and this
-                    // read. Omit it — reconcile is best-effort and the node retries next round.
-                    _logger.LogDebug(
-                        "Reconcile: pending row for instance {Instance} disappeared between stage and read (purged race); omitting from gap",
-                        exp.InstanceUniqueName);
-                }
-                continue;
-            }

-            gap.Add(new ReconcileGapItem(
-                exp.InstanceUniqueName, snapshot.DeploymentId, exp.RevisionHash, exp.IsEnabled, token));
+                gap.Add(new ReconcileGapItem(
+                    exp.InstanceUniqueName, row.DeploymentId, row.RevisionHash, exp.IsEnabled, row.Token));
+            }
        }

        // 5. ORPHANS = names the node has locally that central no longer considers deployed.
@@ -573,6 +573,145 @@ public class DeploymentManagerRepository : IDeploymentManagerRepository
        return !liveExists;
    }

+    /// <inheritdoc />
+    public async Task<IReadOnlyDictionary<int, DeployedConfigSnapshot>> GetDeployedSnapshotsByInstanceIdsAsync(
+        IReadOnlyCollection<int> instanceIds, CancellationToken cancellationToken = default)
+    {
+        ArgumentNullException.ThrowIfNull(instanceIds);
+        if (instanceIds.Count == 0)
+            return new Dictionary<int, DeployedConfigSnapshot>();
+
+        var ids = instanceIds.Distinct().ToArray();
+        return await _dbContext.Set<DeployedConfigSnapshot>()
+            .AsNoTracking()
+            .Where(s => ids.Contains(s.InstanceId))
+            .ToDictionaryAsync(s => s.InstanceId, cancellationToken);
+    }
+
+    /// <inheritdoc />
+    public async Task<IReadOnlyDictionary<int, PendingDeployment>> StagePendingBatchIfAbsentAsync(
+        IReadOnlyList<PendingDeployment> candidates, DateTimeOffset nowUtc,
+        CancellationToken cancellationToken = default)
+    {
+        ArgumentNullException.ThrowIfNull(candidates);
+        var byInstance = new Dictionary<int, PendingDeployment>();
+        foreach (var candidate in candidates)
+            byInstance.TryAdd(candidate.InstanceId, candidate);
+        if (byInstance.Count == 0)
+            return byInstance;
+
+        var ids = byInstance.Keys.ToArray();
+
+        // Same two-step contract as StagePendingIfAbsentAsync, set-based: drop EXPIRED rows
+        // first (they would otherwise read as in-flight and collide on the DeploymentId unique
+        // index when a snapshot's own DeploymentId is re-staged), then insert only where no
+        // LIVE row exists. A live row is never superseded.
+        await DeletePendingWhereAsync(
+            p => ids.Contains(p.InstanceId) && p.ExpiresAtUtc <= nowUtc, cancellationToken);
+
+        var live = await ReadLivePendingAsync(ids, nowUtc, cancellationToken);
+        var added = new List<PendingDeployment>();
+        foreach (var (instanceId, candidate) in byInstance)
+        {
+            if (live.ContainsKey(instanceId))
+                continue;
+            added.Add(candidate);
+        }
+
+        if (added.Count == 0)
+            return live;
+
+        await _dbContext.Set<PendingDeployment>().AddRangeAsync(added, cancellationToken);
+        try
+        {
+            await _dbContext.SaveChangesAsync(cancellationToken);
+        }
+        catch (DbUpdateException)
+        {
+            // The batch lost a race — most commonly the peer site node's concurrent reconcile
+            // inserted the same snapshot DeploymentId first (unique index), or an instance was
+            // deleted mid-reconcile (FK). Un-track the failed inserts and fall back to staging
+            // one row at a time, so one bad row costs only itself and a row the peer staged is
+            // picked up as the live row it now is.
+            foreach (var row in added)
+                _dbContext.Entry(row).State = EntityState.Detached;
+
+            return await StagePendingOneByOneAsync(byInstance, nowUtc, cancellationToken);
+        }
+
+        foreach (var row in added)
+            live[row.InstanceId] = row;
+        return live;
+    }
+
+    /// <summary>
+    /// Fallback for <see cref="StagePendingBatchIfAbsentAsync"/> after its single save failed:
+    /// stages each candidate through <see cref="StagePendingIfAbsentAsync"/>, skipping (and
+    /// omitting from the result) any single row that still cannot be written.
+    /// </summary>
+    private async Task<Dictionary<int, PendingDeployment>> StagePendingOneByOneAsync(
+        IReadOnlyDictionary<int, PendingDeployment> byInstance, DateTimeOffset nowUtc,
+        CancellationToken cancellationToken)
+    {
+        var result = new Dictionary<int, PendingDeployment>();
+        foreach (var (instanceId, candidate) in byInstance)
+        {
+            bool staged;
+            try
+            {
+                staged = await StagePendingIfAbsentAsync(
+                    instanceId, candidate.DeploymentId, candidate.RevisionHash,
+                    candidate.ConfigurationJson, candidate.Token,
+                    nowUtc, candidate.ExpiresAtUtc, cancellationToken);
+            }
+            catch (DbUpdateException)
+            {
+                DetachPendingChanges();
+                continue;
+            }
+
+            if (staged)
+            {
+                result[instanceId] = candidate;
+                continue;
+            }
+
+            var existing = await ReadLivePendingAsync([instanceId], nowUtc, cancellationToken);
+            if (existing.TryGetValue(instanceId, out var row))
+                result[instanceId] = row;
+        }
+
+        return result;
+    }
+
+    /// <summary>
+    /// Reads the most recent LIVE pending row per instance, projected WITHOUT
+    /// <c>ConfigurationJson</c> (the caller only needs the fetch coordinates), untracked.
+    /// </summary>
+    private async Task<Dictionary<int, PendingDeployment>> ReadLivePendingAsync(
+        int[] instanceIds, DateTimeOffset nowUtc, CancellationToken cancellationToken)
+    {
+        var rows = await _dbContext.Set<PendingDeployment>()
+            .AsNoTracking()
+            .Where(p => instanceIds.Contains(p.InstanceId) && p.ExpiresAtUtc > nowUtc)
+            .Select(p => new
+            {
+                p.Id, p.DeploymentId, p.InstanceId, p.RevisionHash, p.Token, p.CreatedAtUtc, p.ExpiresAtUtc
+            })
+            .ToListAsync(cancellationToken);
+
+        // At most one live row per instance by design; on a hypothetical duplicate take the
+        // most recent, mirroring GetPendingDeploymentByInstanceIdAsync's ordering.
+        return rows
+            .GroupBy(r => r.InstanceId)
+            .Select(g => g.OrderByDescending(r => r.CreatedAtUtc).ThenByDescending(r => r.Id).First())
+            .ToDictionary(
+                r => r.InstanceId,
+                r => new PendingDeployment(
+                    r.DeploymentId, r.InstanceId, r.RevisionHash, string.Empty, r.Token,
+                    r.CreatedAtUtc, r.ExpiresAtUtc) { Id = r.Id });
+    }
+
    // --- Instance lookups for deployment pipeline ---

    /// <inheritdoc />
@@ -8,8 +8,8 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;

 /// <summary>
 /// Runs on EVERY site node (NOT a singleton) so a standby that was DOWN during a deploy
-/// self-heals on its next restart. On startup the actor performs one best-effort
-/// reconciliation pass:
+/// self-heals on its next restart. On startup the actor performs a best-effort
+/// reconciliation pass (retried a bounded number of times, see remarks):
 /// <list type="number">
 ///   <item>read the node's local deployed inventory from SQLite,</item>
 ///   <item>report it to central via the <c>SiteCommunicationActor</c> Ask
@@ -23,10 +23,18 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
 /// <remarks>
 /// <para>
 /// <b>Best-effort throughout.</b> A central-unreachable / timed-out Ask is caught, logged
-/// at Warning, and the pass simply ends — reconcile re-runs on the next node startup; it is a
-/// self-heal, not a critical path. A per-item fetch/write failure is caught and logged, then
-/// the remaining gap items continue (one bad item must not abort the rest). The actor never
-/// crashes on these failures.
+/// at Warning, and the pass ends; it is a self-heal, not a critical path. A per-item
+/// fetch/write failure is caught and logged, then the remaining gap items continue (one bad
+/// item must not abort the rest). The actor never crashes on these failures.
+/// </para>
+/// <para>
+/// <b>Bounded retry.</b> A pass that did not complete (central unreachable, Ask timeout,
+/// local read error) or completed with failed items is re-run, up to <c>maxAttempts</c>
+/// passes in total, after an exponential back-off (<c>retryBaseDelay</c>, then 2x, 4x...).
+/// Each retry re-reads the local inventory, so items an earlier pass healed are no longer in
+/// the gap. Before this, a failed pass waited for the NEXT node restart — leaving exactly the
+/// node this actor exists to heal unhealed. After the last attempt the actor logs and stops
+/// trying until the next startup.
 /// </para>
 /// <para>
 /// The pass runs after a small startup delay (so the site's central transport has time to
@@ -63,6 +71,12 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
    private readonly ILogger<SiteReconciliationActor> _logger;
    private readonly TimeSpan _initialDelay;
    private readonly TimeSpan _askTimeout;
+    private readonly int _maxAttempts;
+    private readonly TimeSpan _retryBaseDelay;
+
+    // Number of passes started so far (1-based once the first pass runs). Only touched on
+    // the actor thread.
+    private int _attempt;

    /// <summary>Akka timer scheduler injected by the framework via <see cref="IWithTimers"/>.</summary>
    public ITimerScheduler Timers { get; set; } = null!;
@@ -93,6 +107,13 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
    /// node's reconcile pass found the gap. Optional: when null the pass stores configs and
    /// starts nothing, exactly as it did before.
    /// </param>
+    /// <param name="maxAttempts">
+    /// Total passes allowed per actor lifetime when a pass fails or leaves failed items.
+    /// Defaults to 3; values below 1 are treated as 1 (no retry).
+    /// </param>
+    /// <param name="retryBaseDelay">
+    /// Back-off before the first retry, doubled for each later one. Defaults to 30 seconds.
+    /// </param>
    public SiteReconciliationActor(
        SiteStorageService storage,
        IDeploymentConfigFetcher configFetcher,
@@ -102,7 +123,9 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
        ILogger<SiteReconciliationActor> logger,
        TimeSpan? initialDelay = null,
        TimeSpan? askTimeout = null,
-        IActorRef? deploymentManager = null)
+        IActorRef? deploymentManager = null,
+        int maxAttempts = 3,
+        TimeSpan? retryBaseDelay = null)
    {
        _storage = storage;
        _configFetcher = configFetcher;
@@ -113,23 +136,33 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
        _logger = logger;
        _initialDelay = initialDelay ?? TimeSpan.FromSeconds(5);
        _askTimeout = askTimeout ?? TimeSpan.FromSeconds(30);
+        _maxAttempts = Math.Max(1, maxAttempts);
+        _retryBaseDelay = retryBaseDelay ?? TimeSpan.FromSeconds(30);

-        Receive<RunReconcile>(_ => RunReconcilePassAsync().PipeTo(Self));
+        Receive<RunReconcile>(_ =>
+        {
+            _attempt++;
+            RunReconcilePassAsync().PipeTo(Self);
+        });
        Receive<ReconcilePassResult>(HandleReconcilePassResult);

        // Defensive: RunReconcilePassAsync is designed never to throw (it returns a faulted
        // ReconcilePassResult instead), but if anything unexpected faults the piped Task the
        // Status.Failure would otherwise go to dead letters silently. Log it instead.
-        Receive<Status.Failure>(f => _logger.LogWarning(f.Cause,
-            "Reconcile pass faulted unexpectedly for site {Site} node {Node}",
-            _siteIdentifier, _nodeId));
+        Receive<Status.Failure>(f =>
+        {
+            _logger.LogWarning(f.Cause,
+                "Reconcile pass faulted unexpectedly for site {Site} node {Node}",
+                _siteIdentifier, _nodeId);
+            ScheduleRetryIfAllowed();
+        });
    }

    /// <inheritdoc />
    protected override void PreStart()
    {
        base.PreStart();
-        // One-shot pass after a small delay so the site's central transport can connect first.
+        // First pass after a small delay so the site's central transport can connect first.
        // Non-blocking: the timer fires RunReconcile back onto this actor's mailbox.
        Timers.StartSingleTimer(StartupTimerKey, RunReconcile.Instance, _initialDelay);
        _logger.LogInformation(
@@ -239,16 +272,24 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
        if (result.Error != null)
        {
            // Best-effort: a failed pass (central unreachable, Ask timeout, local read error) is
-            // logged at Warning and the actor stays alive. Reconcile re-runs on the next startup.
+            // logged at Warning and the actor stays alive; it is retried with back-off up to the
+            // attempt cap, then left to the next startup.
            _logger.LogWarning(result.Error,
-                "Reconcile pass for site {Site} node {Node} did not complete (central unreachable or read error) — will retry on next startup",
-                _siteIdentifier, _nodeId);
+                "Reconcile pass {Attempt}/{MaxAttempts} for site {Site} node {Node} did not complete (central unreachable or read error)",
+                _attempt, _maxAttempts, _siteIdentifier, _nodeId);
+            ScheduleRetryIfAllowed();
            return;
        }

        _logger.LogInformation(
-            "Reconcile pass for site {Site} node {Node} complete: {Fetched} fetched, {Failed} failed, {Orphans} orphan(s)",
-            _siteIdentifier, _nodeId, result.Fetched, result.Failed, result.Orphans);
+            "Reconcile pass {Attempt}/{MaxAttempts} for site {Site} node {Node} complete: {Fetched} fetched, {Failed} failed, {Orphans} orphan(s)",
+            _attempt, _maxAttempts, _siteIdentifier, _nodeId, result.Fetched, result.Failed, result.Orphans);
+
+        // Items that failed to fetch/store are still missing: retry the pass (the re-read
+        // inventory makes the gap only what is still missing). The healed instances below are
+        // started now regardless — a partial pass still healed them.
+        if (result.Failed > 0)
+            ScheduleRetryIfAllowed();

        // Heal the RUNTIME, not just the file (row 67). Storing a config starts nothing: the
        // Deployment Manager's boot-time SQLite load has already run by now, so without this
@@ -268,9 +309,30 @@ public sealed class SiteReconciliationActor : ReceiveActor, IWithTimers
        }
    }

+    /// <summary>
+    /// Schedules the next reconcile pass after an exponential back-off, unless the attempt cap
+    /// is reached — in which case the actor logs and leaves the heal to the next node startup.
+    /// </summary>
+    private void ScheduleRetryIfAllowed()
+    {
+        if (_attempt >= _maxAttempts)
+        {
+            _logger.LogWarning(
+                "Reconcile for site {Site} node {Node} gave up after {Attempts} attempt(s) — will retry on next startup",
+                _siteIdentifier, _nodeId, _attempt);
+            return;
+        }
+
+        var delay = TimeSpan.FromTicks(_retryBaseDelay.Ticks * (1L << Math.Min(_attempt - 1, 16)));
+        _logger.LogInformation(
+            "Reconcile for site {Site} node {Node}: retrying in {Delay} (attempt {Next}/{MaxAttempts})",
+            _siteIdentifier, _nodeId, delay, _attempt + 1, _maxAttempts);
+        Timers.StartSingleTimer(StartupTimerKey, RunReconcile.Instance, delay);
+    }
+
    // ── Internal messages ──

-    /// <summary>Self-tick that drives the one-shot startup reconcile pass.</summary>
+    /// <summary>Self-tick that drives a startup reconcile pass (the first, or a retry).</summary>
    private sealed class RunReconcile
    {
        public static readonly RunReconcile Instance = new();