dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(reconcile): batch startup reconcile into fixed round trips, bound central work by QueryTimeout, retry failed site passes with back-off (review 2026-10-21 03/P-1)

Browse Source
This commit is contained in:
Joseph Doherty
2026-10-22 06:25:57 -04:00
parent d6a9a79c27
commit 7c83a4aa94
9 changed files with 638 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.Communication/ReconcileService.cs
+75 -65
View File
@@ -1,5 +1,6 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Deployment;
@@ -18,6 +19,11 @@ namespace ZB.MOM.WW.ScadaBridge.Communication;
/// replicate. The node fetches each gap item's config over the existing token-gated HTTP
/// endpoint; this service stages those tokens as <c>PendingDeployment</c> rows.
///
/// The whole pass is a fixed number of round trips regardless of gap size: the expected-set
/// projection, ONE untracked read of the gap's snapshot configs, and ONE batched
/// insert-if-absent of the pending rows. Work is bounded by
/// <see cref="CommunicationOptions.QueryTimeout"/> — the site's own deadline for the reply.
///
/// Scoped service (holds scoped repositories); resolved per-request by the actor inside a
/// DI scope. The actor stays thin: it captures <c>Sender</c>, resolves this service in a
/// scope, awaits <see cref="ReconcileAsync"/>, and pipes the response back.
@@ -73,92 +79,96 @@ public class ReconcileService
Array.Empty<ReconcileGapItem>(), Array.Empty<string>(), baseUrl);
}
// 2. Central's expected deployed set for this site (instances with a snapshot).
// Central does no more work than the site will wait for: the node's own Ask/gRPC
// deadline is QueryTimeout, so past it every staged token would go to a caller that
// has already given up. (The actor-lifetime token passed in is not request-scoped.)
using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
if (_options.QueryTimeout > TimeSpan.Zero)
deadline.CancelAfter(_options.QueryTimeout);
var ct = deadline.Token;
// 2. Central's expected deployed set for this site (instances with a snapshot) — a
// projection carrying each snapshot's DeploymentId/RevisionHash, no config JSON.
var expected = await _deploymentRepository
.GetExpectedDeploymentsForSiteAsync(site.Id, cancellationToken)
.GetExpectedDeploymentsForSiteAsync(site.Id, ct)
.ConfigureAwait(false);
var localMap = request.LocalNameToRevisionHash;
var gap = new List<ReconcileGapItem>();
var now = DateTimeOffset.UtcNow;
var expiresAt = now + _options.PendingDeploymentTtl;
// 3. GAP = expected items the node is MISSING (name absent locally) or STALE
// (local revision != expected revision). Current items are omitted.
foreach (var exp in expected)
{
var present = localMap.TryGetValue(exp.InstanceUniqueName, out var localHash);
var stale = present && !string.Equals(localHash, exp.RevisionHash, StringComparison.Ordinal);
if (present && !stale)
continue; // node already has the current revision
var gapExpected = expected
.Where(exp => !localMap.TryGetValue(exp.InstanceUniqueName, out var localHash)
|| !string.Equals(localHash, exp.RevisionHash, StringComparison.Ordinal))
.ToList();
// 4. Read the frozen snapshot config to stage. Null = the snapshot was deleted
// between the expected-set query and now (instance removed mid-reconcile);
// skip it — best-effort reconcile re-runs on the next node startup.
var snapshot = await _deploymentRepository
.GetDeployedSnapshotByInstanceIdAsync(exp.InstanceId, cancellationToken)
var gap = new List<ReconcileGapItem>(gapExpected.Count);
if (gapExpected.Count > 0)
{
// 4. ONE read of the frozen snapshot configs, for the gap only — this is the only
// query that materialises ConfigurationJson. A missing entry means the snapshot
// was deleted between the expected-set query and now (instance removed
// mid-reconcile); it is skipped.
var snapshots = await _deploymentRepository
.GetDeployedSnapshotsByInstanceIdsAsync(
gapExpected.Select(e => e.InstanceId).ToList(), ct)
.ConfigureAwait(false);
if (snapshot == null)
var now = DateTimeOffset.UtcNow;
var expiresAt = now + _options.PendingDeploymentTtl;
// Stage with the snapshot's DeploymentId so the gap item's DeploymentId + token point
// the node at the right pending row to fetch.
var staging = new List<(ExpectedDeployment Expected, PendingDeployment Candidate)>(gapExpected.Count);
foreach (var exp in gapExpected)
{
_logger.LogDebug(
"Reconcile: snapshot for instance {Instance} disappeared (deleted race); skipping",
exp.InstanceUniqueName);
continue;
if (!snapshots.TryGetValue(exp.InstanceId, out var snapshot))
{
_logger.LogDebug(
"Reconcile: snapshot for instance {Instance} disappeared (deleted race); skipping",
exp.InstanceUniqueName);
continue;
}
staging.Add((exp, new PendingDeployment(
snapshot.DeploymentId, exp.InstanceId, snapshot.RevisionHash,
snapshot.ConfigurationJson, DeploymentFetchToken.Generate(), now, expiresAt)));
}
var token = DeploymentFetchToken.Generate();
// Stage with the snapshot's DeploymentId as the deploymentId so the gap item's
// DeploymentId + token point the node at the right pending row to fetch.
//
// StagePendingIfAbsent is insert-if-absent: if BOTH site nodes are concurrently
// missing the same instance (e.g. fresh container start / cleared SQLite after a
// successful deploy), both attempt to stage here. The first succeeds (true); the
// second gets false and is handled in the !staged branch below — it returns the
// existing pending row's token so it heals in the same round, rather than being
// omitted. Deploy-time supersession serializes via the per-instance operation lock.
var staged = await _deploymentRepository.StagePendingIfAbsentAsync(
exp.InstanceId, snapshot.DeploymentId, exp.RevisionHash,
snapshot.ConfigurationJson, token, now, expiresAt, cancellationToken)
// Insert-if-absent, in one save. If BOTH site nodes are concurrently missing the same
// instance (fresh container start / cleared SQLite after a successful deploy), both
// reconcile here; the loser finds the winner's LIVE row and is handed ITS
// DeploymentId/RevisionHash/Token (the fetch token is multi-use within its TTL), so
// both nodes heal in the same round instead of the second waiting for a restart. The
// same applies to an in-flight deploy's row: its config is newer than the snapshot,
// and fetching it is still correct (the site's guarded write handles ordering).
// Deploy-time supersession serializes via the per-instance operation lock.
var liveRows = await _deploymentRepository
.StagePendingBatchIfAbsentAsync(staging.Select(x => x.Candidate).ToList(), now, ct)
.ConfigureAwait(false);
if (!staged)
foreach (var (exp, candidate) in staging)
{
// A pending row already exists for this instance — either a CONCURRENT reconcile
// from the other site node (both nodes' SQLite empty after a fresh/cleared deploy,
// both reconciling at startup) or an in-flight deploy. Do NOT omit the item: if we
// did, the second concurrently-missing node would get 0 fetched and stay unhealed
// until a later restart. Instead, read the EXISTING pending row and emit a gap item
// carrying ITS DeploymentId/RevisionHash/Token. The fetch token is multi-use within
// its TTL, so both nodes fetch the same pending config and heal in the same round.
// (If the existing row is from an in-flight deploy its config is newer than the
// snapshot — fetching it is still correct; the site's guarded write handles ordering.)
var existing = await _deploymentRepository
.GetPendingDeploymentByInstanceIdAsync(exp.InstanceId, now, cancellationToken)
.ConfigureAwait(false);
if (existing != null)
if (!liveRows.TryGetValue(candidate.InstanceId, out var row))
{
// Raced away: no live row could be staged or found (e.g. the instance was
// deleted mid-reconcile). Omit it — the node retries on its next pass.
_logger.LogDebug(
"Reconcile: no pending row could be staged for instance {Instance}; omitting from gap",
exp.InstanceUniqueName);
continue;
}
if (!ReferenceEquals(row, candidate))
{
_logger.LogDebug(
"Reconcile: pending row already exists for instance {Instance} (concurrent reconcile or in-flight deploy); returning existing token so this node heals too",
exp.InstanceUniqueName);
gap.Add(new ReconcileGapItem(
exp.InstanceUniqueName, existing.DeploymentId, existing.RevisionHash,
exp.IsEnabled, existing.Token));
}
else
{
// Raced away: the pending row was purged between the stage attempt and this
// read. Omit it — reconcile is best-effort and the node retries next round.
_logger.LogDebug(
"Reconcile: pending row for instance {Instance} disappeared between stage and read (purged race); omitting from gap",
exp.InstanceUniqueName);
}
continue;
}
gap.Add(new ReconcileGapItem(
exp.InstanceUniqueName, snapshot.DeploymentId, exp.RevisionHash, exp.IsEnabled, token));
gap.Add(new ReconcileGapItem(
exp.InstanceUniqueName, row.DeploymentId, row.RevisionHash, exp.IsEnabled, row.Token));
}
}
// 5. ORPHANS = names the node has locally that central no longer considers deployed.