using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment; using ZB.MOM.WW.ScadaBridge.Commons.Types.Deployment; namespace ZB.MOM.WW.ScadaBridge.Communication; /// /// Central-side startup-reconciliation handler. A site node, on startup, reports its /// local deployed inventory via (delivered over the /// control plane to ); this service diffs /// it against central's expected deployed set and replies with fresh fetch tokens for the /// gap — instances the node is missing or has at a stale revision — plus the orphan names /// (present locally but no longer deployed centrally, which the node only logs). /// /// This self-heals a node that was DOWN during a deploy and therefore missed the /// replicate. The node fetches each gap item's config over the existing token-gated HTTP /// endpoint; this service stages those tokens as PendingDeployment rows. /// /// The whole pass is a fixed number of round trips regardless of gap size: the expected-set /// projection, ONE untracked read of the gap's snapshot configs, and ONE batched /// insert-if-absent of the pending rows. Work is bounded by /// — the site's own deadline for the reply. /// /// Scoped service (holds scoped repositories); resolved per-request by the actor inside a /// DI scope. The actor stays thin: it captures Sender, resolves this service in a /// scope, awaits , and pipes the response back. /// public class ReconcileService { private readonly IDeploymentManagerRepository _deploymentRepository; private readonly ISiteRepository _siteRepository; private readonly CommunicationOptions _options; private readonly ILogger _logger; /// Initializes the reconciliation service. /// Repository for the expected-set query and pending-row staging. /// Repository used to resolve a site's numeric id from its identifier. /// Communication options carrying and . /// Logger. public ReconcileService( IDeploymentManagerRepository deploymentRepository, ISiteRepository siteRepository, IOptions options, ILogger logger) { _deploymentRepository = deploymentRepository; _siteRepository = siteRepository; _options = options.Value; _logger = logger; } /// /// Diffs the node's reported inventory against central's expected deployed set, /// stages fresh fetch tokens for the gap, and returns the reconcile response. /// /// The site node's reconcile request (its local name→revision-hash map). /// A cancellation token. /// The gap (with fresh tokens), orphan names, and the central fetch base URL. public async Task ReconcileAsync( ReconcileSiteRequest request, CancellationToken cancellationToken = default) { var baseUrl = _options.CentralFetchBaseUrl; // 1. Resolve the numeric site id. An unknown site is non-fatal: reply empty // (the node simply finds no gap to fetch) and log a warning so a // misconfigured SiteIdentifier is visible to operators. var site = await _siteRepository .GetSiteByIdentifierAsync(request.SiteIdentifier, cancellationToken) .ConfigureAwait(false); if (site == null) { _logger.LogWarning( "Reconcile request from unknown site '{SiteIdentifier}' (node {NodeId}); replying with empty gap", request.SiteIdentifier, request.NodeId); return new ReconcileSiteResponse( Array.Empty(), Array.Empty(), baseUrl); } // Central does no more work than the site will wait for: the node's own Ask/gRPC // deadline is QueryTimeout, so past it every staged token would go to a caller that // has already given up. (The actor-lifetime token passed in is not request-scoped.) using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); if (_options.QueryTimeout > TimeSpan.Zero) deadline.CancelAfter(_options.QueryTimeout); var ct = deadline.Token; // 2. Central's expected deployed set for this site (instances with a snapshot) — a // projection carrying each snapshot's DeploymentId/RevisionHash, no config JSON. var expected = await _deploymentRepository .GetExpectedDeploymentsForSiteAsync(site.Id, ct) .ConfigureAwait(false); var localMap = request.LocalNameToRevisionHash; // 3. GAP = expected items the node is MISSING (name absent locally) or STALE // (local revision != expected revision). Current items are omitted. var gapExpected = expected .Where(exp => !localMap.TryGetValue(exp.InstanceUniqueName, out var localHash) || !string.Equals(localHash, exp.RevisionHash, StringComparison.Ordinal)) .ToList(); var gap = new List(gapExpected.Count); if (gapExpected.Count > 0) { // 4. ONE read of the frozen snapshot configs, for the gap only — this is the only // query that materialises ConfigurationJson. A missing entry means the snapshot // was deleted between the expected-set query and now (instance removed // mid-reconcile); it is skipped. var snapshots = await _deploymentRepository .GetDeployedSnapshotsByInstanceIdsAsync( gapExpected.Select(e => e.InstanceId).ToList(), ct) .ConfigureAwait(false); var now = DateTimeOffset.UtcNow; var expiresAt = now + _options.PendingDeploymentTtl; // Stage with the snapshot's DeploymentId so the gap item's DeploymentId + token point // the node at the right pending row to fetch. var staging = new List<(ExpectedDeployment Expected, PendingDeployment Candidate)>(gapExpected.Count); foreach (var exp in gapExpected) { if (!snapshots.TryGetValue(exp.InstanceId, out var snapshot)) { _logger.LogDebug( "Reconcile: snapshot for instance {Instance} disappeared (deleted race); skipping", exp.InstanceUniqueName); continue; } staging.Add((exp, new PendingDeployment( snapshot.DeploymentId, exp.InstanceId, snapshot.RevisionHash, snapshot.ConfigurationJson, DeploymentFetchToken.Generate(), now, expiresAt))); } // Insert-if-absent, in one save. If BOTH site nodes are concurrently missing the same // instance (fresh container start / cleared SQLite after a successful deploy), both // reconcile here; the loser finds the winner's LIVE row and is handed ITS // DeploymentId/RevisionHash/Token (the fetch token is multi-use within its TTL), so // both nodes heal in the same round instead of the second waiting for a restart. The // same applies to an in-flight deploy's row: its config is newer than the snapshot, // and fetching it is still correct (the site's guarded write handles ordering). // Deploy-time supersession serializes via the per-instance operation lock. var liveRows = await _deploymentRepository .StagePendingBatchIfAbsentAsync(staging.Select(x => x.Candidate).ToList(), now, ct) .ConfigureAwait(false); foreach (var (exp, candidate) in staging) { if (!liveRows.TryGetValue(candidate.InstanceId, out var row)) { // Raced away: no live row could be staged or found (e.g. the instance was // deleted mid-reconcile). Omit it — the node retries on its next pass. _logger.LogDebug( "Reconcile: no pending row could be staged for instance {Instance}; omitting from gap", exp.InstanceUniqueName); continue; } if (!ReferenceEquals(row, candidate)) { _logger.LogDebug( "Reconcile: pending row already exists for instance {Instance} (concurrent reconcile or in-flight deploy); returning existing token so this node heals too", exp.InstanceUniqueName); } gap.Add(new ReconcileGapItem( exp.InstanceUniqueName, row.DeploymentId, row.RevisionHash, exp.IsEnabled, row.Token)); } } // 5. ORPHANS = names the node has locally that central no longer considers deployed. // The node only LOGS these (never deletes). var expectedNames = new HashSet( expected.Select(e => e.InstanceUniqueName), StringComparer.Ordinal); var orphans = localMap.Keys .Where(name => !expectedNames.Contains(name)) .ToList(); _logger.LogDebug( "Reconcile for site {SiteIdentifier} (node {NodeId}): {GapCount} gap, {OrphanCount} orphan(s)", request.SiteIdentifier, request.NodeId, gap.Count, orphans.Count); // 6. Reply. return new ReconcileSiteResponse(gap, orphans, baseUrl); } }