fix(reconcile): batch startup reconcile into fixed round trips, bound central work by QueryTimeout, retry failed site passes with back-off (review 2026-10-21 03/P-1)

2026-10-22 06:25:57 -04:00
parent d6a9a79c27
commit 7c83a4aa94
9 changed files with 638 additions and 130 deletions
@@ -573,6 +573,145 @@ public class DeploymentManagerRepository : IDeploymentManagerRepository
        return !liveExists;
    }

+    /// <inheritdoc />
+    public async Task<IReadOnlyDictionary<int, DeployedConfigSnapshot>> GetDeployedSnapshotsByInstanceIdsAsync(
+        IReadOnlyCollection<int> instanceIds, CancellationToken cancellationToken = default)
+    {
+        ArgumentNullException.ThrowIfNull(instanceIds);
+        if (instanceIds.Count == 0)
+            return new Dictionary<int, DeployedConfigSnapshot>();
+
+        var ids = instanceIds.Distinct().ToArray();
+        return await _dbContext.Set<DeployedConfigSnapshot>()
+            .AsNoTracking()
+            .Where(s => ids.Contains(s.InstanceId))
+            .ToDictionaryAsync(s => s.InstanceId, cancellationToken);
+    }
+
+    /// <inheritdoc />
+    public async Task<IReadOnlyDictionary<int, PendingDeployment>> StagePendingBatchIfAbsentAsync(
+        IReadOnlyList<PendingDeployment> candidates, DateTimeOffset nowUtc,
+        CancellationToken cancellationToken = default)
+    {
+        ArgumentNullException.ThrowIfNull(candidates);
+        var byInstance = new Dictionary<int, PendingDeployment>();
+        foreach (var candidate in candidates)
+            byInstance.TryAdd(candidate.InstanceId, candidate);
+        if (byInstance.Count == 0)
+            return byInstance;
+
+        var ids = byInstance.Keys.ToArray();
+
+        // Same two-step contract as StagePendingIfAbsentAsync, set-based: drop EXPIRED rows
+        // first (they would otherwise read as in-flight and collide on the DeploymentId unique
+        // index when a snapshot's own DeploymentId is re-staged), then insert only where no
+        // LIVE row exists. A live row is never superseded.
+        await DeletePendingWhereAsync(
+            p => ids.Contains(p.InstanceId) && p.ExpiresAtUtc <= nowUtc, cancellationToken);
+
+        var live = await ReadLivePendingAsync(ids, nowUtc, cancellationToken);
+        var added = new List<PendingDeployment>();
+        foreach (var (instanceId, candidate) in byInstance)
+        {
+            if (live.ContainsKey(instanceId))
+                continue;
+            added.Add(candidate);
+        }
+
+        if (added.Count == 0)
+            return live;
+
+        await _dbContext.Set<PendingDeployment>().AddRangeAsync(added, cancellationToken);
+        try
+        {
+            await _dbContext.SaveChangesAsync(cancellationToken);
+        }
+        catch (DbUpdateException)
+        {
+            // The batch lost a race — most commonly the peer site node's concurrent reconcile
+            // inserted the same snapshot DeploymentId first (unique index), or an instance was
+            // deleted mid-reconcile (FK). Un-track the failed inserts and fall back to staging
+            // one row at a time, so one bad row costs only itself and a row the peer staged is
+            // picked up as the live row it now is.
+            foreach (var row in added)
+                _dbContext.Entry(row).State = EntityState.Detached;
+
+            return await StagePendingOneByOneAsync(byInstance, nowUtc, cancellationToken);
+        }
+
+        foreach (var row in added)
+            live[row.InstanceId] = row;
+        return live;
+    }
+
+    /// <summary>
+    /// Fallback for <see cref="StagePendingBatchIfAbsentAsync"/> after its single save failed:
+    /// stages each candidate through <see cref="StagePendingIfAbsentAsync"/>, skipping (and
+    /// omitting from the result) any single row that still cannot be written.
+    /// </summary>
+    private async Task<Dictionary<int, PendingDeployment>> StagePendingOneByOneAsync(
+        IReadOnlyDictionary<int, PendingDeployment> byInstance, DateTimeOffset nowUtc,
+        CancellationToken cancellationToken)
+    {
+        var result = new Dictionary<int, PendingDeployment>();
+        foreach (var (instanceId, candidate) in byInstance)
+        {
+            bool staged;
+            try
+            {
+                staged = await StagePendingIfAbsentAsync(
+                    instanceId, candidate.DeploymentId, candidate.RevisionHash,
+                    candidate.ConfigurationJson, candidate.Token,
+                    nowUtc, candidate.ExpiresAtUtc, cancellationToken);
+            }
+            catch (DbUpdateException)
+            {
+                DetachPendingChanges();
+                continue;
+            }
+
+            if (staged)
+            {
+                result[instanceId] = candidate;
+                continue;
+            }
+
+            var existing = await ReadLivePendingAsync([instanceId], nowUtc, cancellationToken);
+            if (existing.TryGetValue(instanceId, out var row))
+                result[instanceId] = row;
+        }
+
+        return result;
+    }
+
+    /// <summary>
+    /// Reads the most recent LIVE pending row per instance, projected WITHOUT
+    /// <c>ConfigurationJson</c> (the caller only needs the fetch coordinates), untracked.
+    /// </summary>
+    private async Task<Dictionary<int, PendingDeployment>> ReadLivePendingAsync(
+        int[] instanceIds, DateTimeOffset nowUtc, CancellationToken cancellationToken)
+    {
+        var rows = await _dbContext.Set<PendingDeployment>()
+            .AsNoTracking()
+            .Where(p => instanceIds.Contains(p.InstanceId) && p.ExpiresAtUtc > nowUtc)
+            .Select(p => new
+            {
+                p.Id, p.DeploymentId, p.InstanceId, p.RevisionHash, p.Token, p.CreatedAtUtc, p.ExpiresAtUtc
+            })
+            .ToListAsync(cancellationToken);
+
+        // At most one live row per instance by design; on a hypothetical duplicate take the
+        // most recent, mirroring GetPendingDeploymentByInstanceIdAsync's ordering.
+        return rows
+            .GroupBy(r => r.InstanceId)
+            .Select(g => g.OrderByDescending(r => r.CreatedAtUtc).ThenByDescending(r => r.Id).First())
+            .ToDictionary(
+                r => r.InstanceId,
+                r => new PendingDeployment(
+                    r.DeploymentId, r.InstanceId, r.RevisionHash, string.Empty, r.Token,
+                    r.CreatedAtUtc, r.ExpiresAtUtc) { Id = r.Id });
+    }
+
    // --- Instance lookups for deployment pipeline ---

    /// <inheritdoc />