ScadaBridge/src/ZB.MOM.WW.ScadaBridge.Communication/ReconcileService.cs

using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Deployment;

namespace ZB.MOM.WW.ScadaBridge.Communication;

/// <summary>
/// Central-side startup-reconciliation handler. A site node, on startup, reports its
/// local deployed inventory via <see cref="ReconcileSiteRequest"/> (delivered over the
/// control plane to <see cref="Actors.CentralCommunicationActor"/>); this service diffs
/// it against central's expected deployed set and replies with fresh fetch tokens for the
/// gap — instances the node is missing or has at a stale revision — plus the orphan names
/// (present locally but no longer deployed centrally, which the node only logs).
///
/// This self-heals a node that was DOWN during a deploy and therefore missed the
/// replicate. The node fetches each gap item's config over the existing token-gated HTTP
/// endpoint; this service stages those tokens as <c>PendingDeployment</c> rows.
///
/// The whole pass is a fixed number of round trips regardless of gap size: the expected-set
/// projection, ONE untracked read of the gap's snapshot configs, and ONE batched
/// insert-if-absent of the pending rows. Work is bounded by
/// <see cref="CommunicationOptions.QueryTimeout"/> — the site's own deadline for the reply.
///
/// Scoped service (holds scoped repositories); resolved per-request by the actor inside a
/// DI scope. The actor stays thin: it captures <c>Sender</c>, resolves this service in a
/// scope, awaits <see cref="ReconcileAsync"/>, and pipes the response back.
/// </summary>
public class ReconcileService
{
    private readonly IDeploymentManagerRepository _deploymentRepository;
    private readonly ISiteRepository _siteRepository;
    private readonly CommunicationOptions _options;
    private readonly ILogger<ReconcileService> _logger;

    /// <summary>Initializes the reconciliation service.</summary>
    /// <param name="deploymentRepository">Repository for the expected-set query and pending-row staging.</param>
    /// <param name="siteRepository">Repository used to resolve a site's numeric id from its identifier.</param>
    /// <param name="options">Communication options carrying <see cref="CommunicationOptions.CentralFetchBaseUrl"/> and <see cref="CommunicationOptions.PendingDeploymentTtl"/>.</param>
    /// <param name="logger">Logger.</param>
    public ReconcileService(
        IDeploymentManagerRepository deploymentRepository,
        ISiteRepository siteRepository,
        IOptions<CommunicationOptions> options,
        ILogger<ReconcileService> logger)
    {
        _deploymentRepository = deploymentRepository;
        _siteRepository = siteRepository;
        _options = options.Value;
        _logger = logger;
    }

    /// <summary>
    /// Diffs the node's reported inventory against central's expected deployed set,
    /// stages fresh fetch tokens for the gap, and returns the reconcile response.
    /// </summary>
    /// <param name="request">The site node's reconcile request (its local name→revision-hash map).</param>
    /// <param name="cancellationToken">A cancellation token.</param>
    /// <returns>The gap (with fresh tokens), orphan names, and the central fetch base URL.</returns>
    public async Task<ReconcileSiteResponse> ReconcileAsync(
        ReconcileSiteRequest request, CancellationToken cancellationToken = default)
    {
        var baseUrl = _options.CentralFetchBaseUrl;

        // 1. Resolve the numeric site id. An unknown site is non-fatal: reply empty
        //    (the node simply finds no gap to fetch) and log a warning so a
        //    misconfigured SiteIdentifier is visible to operators.
        var site = await _siteRepository
            .GetSiteByIdentifierAsync(request.SiteIdentifier, cancellationToken)
            .ConfigureAwait(false);
        if (site == null)
        {
            _logger.LogWarning(
                "Reconcile request from unknown site '{SiteIdentifier}' (node {NodeId}); replying with empty gap",
                request.SiteIdentifier, request.NodeId);
            return new ReconcileSiteResponse(
                Array.Empty<ReconcileGapItem>(), Array.Empty<string>(), baseUrl);
        }

        // Central does no more work than the site will wait for: the node's own Ask/gRPC
        // deadline is QueryTimeout, so past it every staged token would go to a caller that
        // has already given up. (The actor-lifetime token passed in is not request-scoped.)
        using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
        if (_options.QueryTimeout > TimeSpan.Zero)
            deadline.CancelAfter(_options.QueryTimeout);
        var ct = deadline.Token;

        // 2. Central's expected deployed set for this site (instances with a snapshot) — a
        //    projection carrying each snapshot's DeploymentId/RevisionHash, no config JSON.
        var expected = await _deploymentRepository
            .GetExpectedDeploymentsForSiteAsync(site.Id, ct)
            .ConfigureAwait(false);

        var localMap = request.LocalNameToRevisionHash;

        // 3. GAP = expected items the node is MISSING (name absent locally) or STALE
        //    (local revision != expected revision). Current items are omitted.
        var gapExpected = expected
            .Where(exp => !localMap.TryGetValue(exp.InstanceUniqueName, out var localHash)
                          || !string.Equals(localHash, exp.RevisionHash, StringComparison.Ordinal))
            .ToList();

        var gap = new List<ReconcileGapItem>(gapExpected.Count);
        if (gapExpected.Count > 0)
        {
            // 4. ONE read of the frozen snapshot configs, for the gap only — this is the only
            //    query that materialises ConfigurationJson. A missing entry means the snapshot
            //    was deleted between the expected-set query and now (instance removed
            //    mid-reconcile); it is skipped.
            var snapshots = await _deploymentRepository
                .GetDeployedSnapshotsByInstanceIdsAsync(
                    gapExpected.Select(e => e.InstanceId).ToList(), ct)
                .ConfigureAwait(false);

            var now = DateTimeOffset.UtcNow;
            var expiresAt = now + _options.PendingDeploymentTtl;

            // Stage with the snapshot's DeploymentId so the gap item's DeploymentId + token point
            // the node at the right pending row to fetch.
            var staging = new List<(ExpectedDeployment Expected, PendingDeployment Candidate)>(gapExpected.Count);
            foreach (var exp in gapExpected)
            {
                if (!snapshots.TryGetValue(exp.InstanceId, out var snapshot))
                {
                    _logger.LogDebug(
                        "Reconcile: snapshot for instance {Instance} disappeared (deleted race); skipping",
                        exp.InstanceUniqueName);
                    continue;
                }

                staging.Add((exp, new PendingDeployment(
                    snapshot.DeploymentId, exp.InstanceId, snapshot.RevisionHash,
                    snapshot.ConfigurationJson, DeploymentFetchToken.Generate(), now, expiresAt)));
            }

            // Insert-if-absent, in one save. If BOTH site nodes are concurrently missing the same
            // instance (fresh container start / cleared SQLite after a successful deploy), both
            // reconcile here; the loser finds the winner's LIVE row and is handed ITS
            // DeploymentId/RevisionHash/Token (the fetch token is multi-use within its TTL), so
            // both nodes heal in the same round instead of the second waiting for a restart. The
            // same applies to an in-flight deploy's row: its config is newer than the snapshot,
            // and fetching it is still correct (the site's guarded write handles ordering).
            // Deploy-time supersession serializes via the per-instance operation lock.
            var liveRows = await _deploymentRepository
                .StagePendingBatchIfAbsentAsync(staging.Select(x => x.Candidate).ToList(), now, ct)
                .ConfigureAwait(false);

            foreach (var (exp, candidate) in staging)
            {
                if (!liveRows.TryGetValue(candidate.InstanceId, out var row))
                {
                    // Raced away: no live row could be staged or found (e.g. the instance was
                    // deleted mid-reconcile). Omit it — the node retries on its next pass.
                    _logger.LogDebug(
                        "Reconcile: no pending row could be staged for instance {Instance}; omitting from gap",
                        exp.InstanceUniqueName);
                    continue;
                }

                if (!ReferenceEquals(row, candidate))
                {
                    _logger.LogDebug(
                        "Reconcile: pending row already exists for instance {Instance} (concurrent reconcile or in-flight deploy); returning existing token so this node heals too",
                        exp.InstanceUniqueName);
                }

                gap.Add(new ReconcileGapItem(
                    exp.InstanceUniqueName, row.DeploymentId, row.RevisionHash, exp.IsEnabled, row.Token));
            }
        }

        // 5. ORPHANS = names the node has locally that central no longer considers deployed.
        //    The node only LOGS these (never deletes).
        var expectedNames = new HashSet<string>(
            expected.Select(e => e.InstanceUniqueName), StringComparer.Ordinal);
        var orphans = localMap.Keys
            .Where(name => !expectedNames.Contains(name))
            .ToList();

        _logger.LogDebug(
            "Reconcile for site {SiteIdentifier} (node {NodeId}): {GapCount} gap, {OrphanCount} orphan(s)",
            request.SiteIdentifier, request.NodeId, gap.Count, orphans.Count);

        // 6. Reply.
        return new ReconcileSiteResponse(gap, orphans, baseUrl);
    }
}