ScadaBridge/src/ZB.MOM.WW.ScadaBridge.DeploymentManager/ArtifactDeploymentService.cs

using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Secrets;
using ZB.MOM.WW.ScadaBridge.Communication;

namespace ZB.MOM.WW.ScadaBridge.DeploymentManager;

/// <summary>
/// System-wide artifact deployment.
/// Broadcasts artifacts (shared scripts, external systems, DB connections, and
/// data connections) to all sites with per-site tracking.
///
/// Notification lists and SMTP configuration are deliberately NOT shipped to
/// sites: notification delivery is central-only (sites store-and-forward to
/// central and never talk to SMTP), so no notification artifact or SMTP
/// credential is ever distributed to a site. The
/// <see cref="DeployArtifactsCommand"/> still carries the
/// <c>NotificationLists</c>/<c>SmtpConfigurations</c> fields for additive
/// message-contract compatibility, but central never populates them.
///
/// - Successful sites are NOT rolled back on other failures.
/// - Failed sites are retryable individually.
/// - 120s timeout per site.
/// - Cross-site version skew is supported.
/// </summary>
public class ArtifactDeploymentService
{
    private readonly ISiteRepository _siteRepo;
    private readonly IDeploymentManagerRepository _deploymentRepo;
    private readonly ITemplateEngineRepository _templateRepo;
    private readonly IExternalSystemRepository _externalSystemRepo;
    private readonly CommunicationService _communicationService;
    private readonly IAuditService _auditService;
    private readonly DeploymentManagerOptions _options;
    private readonly ILogger<ArtifactDeploymentService> _logger;
    private readonly ArtifactRevisionHashService _hashService;
    private readonly SiteSecretsCollector _secretsCollector;

    /// <summary>
    /// Initializes a new instance of the ArtifactDeploymentService.
    /// </summary>
    /// <param name="siteRepo">Repository for site data.</param>
    /// <param name="deploymentRepo">Repository for deployment records.</param>
    /// <param name="templateRepo">Repository for templates.</param>
    /// <param name="externalSystemRepo">Repository for external systems.</param>
    /// <param name="notificationRepo">
    /// Retained on the signature for DI/source compatibility but
    /// intentionally NOT consumed. Notification lists and SMTP configuration are
    /// central-only and are never shipped to sites, so the artifact path must not read
    /// the notification repository at all.
    /// </param>
    /// <param name="communicationService">Service for communicating with sites.</param>
    /// <param name="auditService">Service for audit logging.</param>
    /// <param name="options">Deployment manager options.</param>
    /// <param name="logger">Logger instance.</param>
    /// <param name="hashService">
    /// Computes the deterministic revision hash of a per-site artifact set, stamped on the
    /// <see cref="Site"/> row after a successful deploy. Second-to-last on the signature so
    /// the addition stayed positionally additive for existing call sites at the time it was
    /// introduced.
    /// </param>
    /// <param name="secretsCollector">
    /// Resolves a site's referenced secrets (ciphertext rows) against the central secret
    /// store (design §3.1/§3.3). Last on the signature for the same reason.
    /// </param>
    public ArtifactDeploymentService(
        ISiteRepository siteRepo,
        IDeploymentManagerRepository deploymentRepo,
        ITemplateEngineRepository templateRepo,
        IExternalSystemRepository externalSystemRepo,
        INotificationRepository notificationRepo,
        CommunicationService communicationService,
        IAuditService auditService,
        IOptions<DeploymentManagerOptions> options,
        ILogger<ArtifactDeploymentService> logger,
        ArtifactRevisionHashService hashService,
        SiteSecretsCollector secretsCollector)
    {
        _siteRepo = siteRepo;
        _deploymentRepo = deploymentRepo;
        _templateRepo = templateRepo;
        _externalSystemRepo = externalSystemRepo;
        // notificationRepo is deliberately not stored — notification
        // lists and SMTP configs are central-only and are never fetched for shipping to sites.
        _ = notificationRepo;
        _communicationService = communicationService;
        _auditService = auditService;
        _options = options.Value;
        _logger = logger;
        _hashService = hashService;
        _secretsCollector = secretsCollector;
    }

    /// <summary>
    /// Collects all artifact types from repositories and builds a <see cref="DeployArtifactsCommand"/>
    /// scoped to a specific site's data connections.
    /// </summary>
    /// <param name="siteId">The DB id of the site whose data connections are collected.</param>
    /// <param name="cancellationToken">Cancellation token.</param>
    /// <param name="deploymentId">
    /// The logical deployment id for this artifact deployment. All per-site
    /// commands of one <see cref="DeployToAllSitesAsync"/> call share this id so the audit log,
    /// UI summary, and persisted record correlate. When <c>null</c> a fresh id is minted (used by
    /// single-site retries).
    /// </param>
    /// <returns>
    /// The deployment artifacts command for the site, and any secret references its data
    /// connections make that could not be resolved at central (absent or tombstoned). A
    /// non-empty <c>Missing</c> list means the command must NOT be sent to the site —
    /// see <see cref="DeployCoreAsync"/>/<see cref="RetryForSiteAsync"/>.
    /// </returns>
    /// <remarks>
    /// This convenience overload runs the global artifact queries
    /// for a single site (used by <see cref="RetryForSiteAsync"/>). The multi-site
    /// <see cref="DeployToAllSitesAsync"/> path hoists the global queries OUT of the
    /// per-site loop and calls the prefetched-globals overload to avoid the N+1
    /// re-query of every system-wide artifact set per site.
    /// </remarks>
    public async Task<(DeployArtifactsCommand Command, IReadOnlyList<SecretReference> Missing)> BuildDeployArtifactsCommandAsync(
        int siteId,
        CancellationToken cancellationToken = default,
        string? deploymentId = null)
    {
        var globals = await FetchGlobalArtifactsAsync(cancellationToken);
        return await BuildDeployArtifactsCommandAsync(siteId, globals, cancellationToken, deploymentId);
    }

    /// <summary>
    /// Builds a per-site <see cref="DeployArtifactsCommand"/> using a previously-fetched
    /// snapshot of the global artifact sets (shared scripts, external systems + methods,
    /// DB connections). Only the per-site data-connection query — and the per-site secret
    /// resolution it drives — runs here.
    /// </summary>
    /// <remarks>
    /// Separating the global fetch from the per-site build lets
    /// <see cref="DeployToAllSitesAsync"/> issue the global queries exactly once across
    /// the whole multi-site sweep, eliminating the N+1 re-query of shared scripts,
    /// external systems, methods, and DB connections.
    ///
    /// The command's <c>NotificationLists</c> and
    /// <c>SmtpConfigurations</c> fields are always sent <c>null</c> — notification
    /// delivery is central-only and no notification artifact or SMTP credential is
    /// ever distributed to a site. The fields remain on the contract only for
    /// additive compatibility.
    ///
    /// The command's <c>Secrets</c>/<c>MissingSecrets</c> fields (design §3.1/§3.3) are
    /// populated from <see cref="SiteSecretsCollector.CollectAsync"/> regardless of whether
    /// any reference is missing — <c>MissingSecrets</c> is CENTRAL-ONLY (never mapped to the
    /// wire) and exists solely so <see cref="ArtifactRevisionHashService"/> can fold a
    /// distinguishable marker into the hash even for a command the caller ultimately refuses
    /// to send (see the returned <c>Missing</c> tuple element).
    /// </remarks>
    private async Task<(DeployArtifactsCommand Command, IReadOnlyList<SecretReference> Missing)> BuildDeployArtifactsCommandAsync(
        int siteId,
        GlobalArtifactSnapshot globals,
        CancellationToken cancellationToken,
        string? deploymentId)
    {
        var dataConnections = await _siteRepo.GetDataConnectionsBySiteIdAsync(siteId, cancellationToken);

        // Map data connections
        var dataConnectionArtifacts = dataConnections.Select(dc =>
            new DataConnectionArtifact(dc.Name, dc.Protocol, dc.PrimaryConfiguration, dc.BackupConfiguration, dc.FailoverRetryCount)).ToList();

        var collected = await _secretsCollector.CollectAsync(dataConnectionArtifacts, cancellationToken);

        var command = new DeployArtifactsCommand(
            deploymentId ?? Guid.NewGuid().ToString("N"),
            globals.SharedScripts,
            globals.ExternalSystems,
            globals.DatabaseConnections,
            // Notification lists are central-only — never shipped to sites.
            NotificationLists: null,
            dataConnectionArtifacts,
            // SMTP config (incl. credentials) is central-only — never shipped to sites.
            SmtpConfigurations: null,
            DateTimeOffset.UtcNow,
            // ALWAYS a list — never null — even when the site references no secrets. The site
            // applies Secrets as the FULL set (every deployed_secrets row absent from it is
            // marked removed and the mirror deletes it), and a null Secrets means "central did
            // not ship secrets at all" (the pre-feature/old-central contract), which touches
            // nothing. Shipping null here when the LAST reference is dropped therefore left the
            // secret live on both site nodes forever (Task 13 live gate, check 5). An empty
            // list costs an old site nothing (it ignores proto field 9), and the hash is
            // unaffected: the hash service folds an empty and a null list identically.
            Secrets: collected.Secrets,
            MissingSecrets: collected.Missing.Count > 0 ? collected.Missing.Select(m => m.Name).ToList() : null);

        return (command, collected.Missing);
    }

    /// <summary>
    /// Task 7 (design §3.4): reconciles a site's raw <see cref="ArtifactDeploymentResponse"/>
    /// against what the deployed <see cref="DeployArtifactsCommand"/> actually shipped, so
    /// central never stamps a site *Current* on the strength of a reply that only proves the
    /// artifact set was RECEIVED, not that every shipped secret was APPLIED.
    ///
    /// A site that predates the secrets feature answers with <c>SecretsApplied = null</c> even
    /// on an otherwise-successful apply — silently ignoring the new proto field rather than
    /// failing loudly — so that case must be caught here rather than trusted as success. A site
    /// that applied some but not all shipped secrets (a partial local failure) reports a count
    /// that undershoots what was sent. Both are downgraded to a failure; the site's previous
    /// stamp (or lack of one) is left untouched because stamping already keys off
    /// <see cref="SiteArtifactResult.Success"/>.
    ///
    /// Only fires when the command actually shipped secrets — a site with nothing to apply
    /// truthfully has nothing to confirm, so its <c>SecretsApplied</c> stays uninterpreted
    /// (old-site behaviour is unchanged when no secrets are in play, per the acceptance
    /// criteria).
    /// </summary>
    private static SiteArtifactResult ReconcileSecretsOutcome(
        DeployArtifactsCommand cmd, ArtifactDeploymentResponse r, string siteIdentifier, string siteName)
    {
        var shipped = cmd.Secrets?.Count ?? 0;
        if (r.Success && shipped > 0 && r.SecretsApplied is null)
            return new(siteIdentifier, siteName, false,
                "Site build predates secret deployment — upgrade the site before deploying secrets", null);
        if (r.Success && shipped > 0 && r.SecretsApplied != shipped)
            return new(siteIdentifier, siteName, false,
                $"Site applied {r.SecretsApplied} of {shipped} secrets", r.SecretsApplied);
        return new(siteIdentifier, siteName, r.Success, r.ErrorMessage, shipped > 0 ? r.SecretsApplied : null);
    }

    /// <summary>
    /// Best-effort <c>DeploySiteSecrets</c> audit row (task 7): one row per site whose command
    /// carried, or would have carried, at least one secret — a resolved
    /// <see cref="DeployArtifactsCommand.Secrets"/> entry, an unresolved reference in
    /// <paramref name="missing"/> (the fail-closed case, where nothing was sent), or both. A
    /// site with no secret references at all gets no row.
    ///
    /// Per the audit-is-best-effort rule (an audit failure never changes the deploy outcome)
    /// this is fire-and-forget-shaped only in intent, not mechanism — like every other audit
    /// call on this path it is still awaited (never on a script path; this is a management
    /// operation), but a thrown/faulted write here must not — and structurally cannot, since
    /// the caller already decided <see cref="SiteArtifactResult.Success"/> beforehand — change
    /// whether the site gets stamped.
    ///
    /// The <c>afterState</c> payload is built ONLY from names/revisions/missing names/success/
    /// error — never <see cref="SecretArtifact.Ciphertext"/>, <c>Nonce</c>, <c>Tag</c>,
    /// <c>WrappedDek</c>, or any other material (global constraint: never plaintext or material
    /// in logs or audit rows).
    /// </summary>
    private Task LogSecretsAuditAsync(
        string user,
        DeployArtifactsCommand command,
        IReadOnlyList<SecretReference> missing,
        string siteIdentifier,
        string siteName,
        bool success,
        string? error,
        CancellationToken cancellationToken)
    {
        var hasSecretRefs = (command.Secrets?.Count ?? 0) > 0 || missing.Count > 0;
        if (!hasSecretRefs)
            return Task.CompletedTask;

        return _auditService.LogAsync(
            user, "DeploySiteSecrets", "SiteSecrets", siteIdentifier, siteName,
            new
            {
                deploymentId = command.DeploymentId,
                secrets = (command.Secrets ?? [])
                    .Select(s => new { name = s.Name, revision = s.Revision })
                    .ToList(),
                missing = missing.Select(m => m.Name).ToList(),
                success,
                error,
            },
            cancellationToken);
    }

    /// <summary>
    /// Formats the fail-closed message for one or more secret references a site's data
    /// connections make that could not be resolved at central (design §3.3). Never includes
    /// secret material — only names and the referencing connection names.
    /// </summary>
    private static string BuildMissingSecretsMessage(IReadOnlyList<SecretReference> missing) =>
        string.Join("; ", missing.Select(m =>
            $"Secret '{m.Name}' referenced by data connection(s) {string.Join(", ", m.ReferencedBy)} " +
            "is not in the central secret store"));

    /// <summary>
    /// Fetches the system-wide artifact sets that are identical across every site —
    /// shared scripts, external systems (with their methods serialized in), and
    /// database connections. Used by <see cref="DeployToAllSitesAsync"/> to pre-load
    /// once before the per-site loop.
    /// </summary>
    /// <remarks>
    /// The per-site artifact build path previously re-issued
    /// every one of these queries per site (≈ N + M·N round trips for N sites
    /// and M external systems). Hoisting them here drops that to a single fetch.
    ///
    /// Notification lists and SMTP configurations are NOT
    /// fetched here. Notification delivery is central-only, so they are never
    /// shipped to sites — the artifact path must not even read them.
    /// </remarks>
    private async Task<GlobalArtifactSnapshot> FetchGlobalArtifactsAsync(CancellationToken cancellationToken)
    {
        var sharedScripts = await _templateRepo.GetAllSharedScriptsAsync(cancellationToken);
        var externalSystems = await _externalSystemRepo.GetAllExternalSystemsAsync(cancellationToken);
        var dbConnections = await _externalSystemRepo.GetAllDatabaseConnectionsAsync(cancellationToken);

        // Map shared scripts
        var scriptArtifacts = sharedScripts.Select(s =>
            new SharedScriptArtifact(s.Name, s.Code, s.ParameterDefinitions, s.ReturnDefinition)).ToList();

        // Map external systems (serialize methods per system)
        var externalSystemArtifacts = new List<ExternalSystemArtifact>();
        foreach (var es in externalSystems)
        {
            var methods = await _externalSystemRepo.GetMethodsByExternalSystemIdAsync(es.Id, cancellationToken);
            // The property NAMES emitted here are part of a contract:
            // ArtifactRevisionHashService.CanonicalizeMethodDefinitions re-orders this array by
            // each element's "Name" property (case-insensitively) so repository row order cannot
            // move a site's artifact revision hash. Renaming Name here would silently degrade
            // that canonicalisation to "leave the order alone".
            var methodsJson = methods.Count > 0
                ? JsonSerializer.Serialize(methods.Select(m => new
                {
                    m.Name,
                    m.HttpMethod,
                    m.Path,
                    m.ParameterDefinitions,
                    m.ReturnDefinition
                }))
                : null;
            // MaxRetries/RetryDelay ride along so the site's Store-and-Forward buffering honours
            // the centrally-configured per-system retry policy. Before they were carried, a site
            // silently fell back to the engine defaults (50 x 30 s) for every cached call, whatever
            // central had configured — the DatabaseConnectionArtifact below always carried them.
            externalSystemArtifacts.Add(new ExternalSystemArtifact(
                es.Name, es.EndpointUrl, es.AuthType, es.AuthConfiguration, methodsJson, es.TimeoutSeconds,
                es.MaxRetries, es.RetryDelay));
        }

        // Map database connections
        var dbConnectionArtifacts = dbConnections.Select(d =>
            new DatabaseConnectionArtifact(d.Name, d.ConnectionString, d.MaxRetries, d.RetryDelay)).ToList();

        return new GlobalArtifactSnapshot(
            scriptArtifacts,
            externalSystemArtifacts,
            dbConnectionArtifacts);
    }

    /// <summary>
    /// Bag of the global artifact sets that do not vary per site, captured once at
    /// the start of <see cref="DeployToAllSitesAsync"/> and reused for every per-site
    /// command build. Notification lists and SMTP
    /// configurations are deliberately absent — they are central-only and never
    /// shipped to sites.
    /// </summary>
    private sealed record GlobalArtifactSnapshot(
        IReadOnlyList<SharedScriptArtifact> SharedScripts,
        IReadOnlyList<ExternalSystemArtifact> ExternalSystems,
        IReadOnlyList<DatabaseConnectionArtifact> DatabaseConnections);

    /// <summary>
    /// Deploys artifacts to all sites. Builds a per-site command with that site's data connections.
    /// Returns per-site result matrix.
    /// </summary>
    /// <param name="user">The user initiating the deployment.</param>
    /// <param name="cancellationToken">Cancellation token.</param>
    /// <returns>A summary of the artifact deployment results for all sites.</returns>
    public virtual async Task<Result<ArtifactDeploymentSummary>> DeployToAllSitesAsync(
        string user,
        CancellationToken cancellationToken = default)
    {
        var sites = await _siteRepo.GetAllSitesAsync(cancellationToken);
        if (sites.Count == 0)
            return Result<ArtifactDeploymentSummary>.Failure("No sites configured.");

        return await DeployCoreAsync(sites, user, cancellationToken);
    }

    /// <summary>
    /// Deploys system-wide artifacts to a single site (management-API --site-id path).
    /// </summary>
    /// <param name="siteId">The database identifier of the target site.</param>
    /// <param name="user">The user initiating the deployment.</param>
    /// <param name="cancellationToken">Cancellation token.</param>
    /// <returns>A summary of the artifact deployment result for the single site.</returns>
    public virtual async Task<Result<ArtifactDeploymentSummary>> DeployToSiteAsync(
        int siteId,
        string user,
        CancellationToken cancellationToken = default)
    {
        var site = await _siteRepo.GetSiteByIdAsync(siteId, cancellationToken);
        if (site is null)
            return Result<ArtifactDeploymentSummary>.Failure($"Site with ID {siteId} not found.");

        return await DeployCoreAsync([site], user, cancellationToken);
    }

    /// <summary>
    /// Core deployment path shared by the all-sites and single-site entry points.
    /// Builds a per-site command with that site's data connections and deploys to
    /// every supplied site under one logical deployment id.
    /// </summary>
    private async Task<Result<ArtifactDeploymentSummary>> DeployCoreAsync(
        IReadOnlyList<Site> sites,
        string user,
        CancellationToken cancellationToken)
    {
        var deploymentId = Guid.NewGuid().ToString("N");
        var perSiteResults = new Dictionary<string, SiteArtifactResult>();

        // Hoist the system-wide artifact queries (shared scripts,
        // external systems + methods, DB connections) OUT of the per-site loop so they
        // run ONCE instead of once per site. Only data connections legitimately vary
        // per site, so they stay inside the loop. (Notification lists and SMTP config
        // are central-only and not fetched at all.)
        var globals = await FetchGlobalArtifactsAsync(cancellationToken);

        // Build per-site commands sequentially (DbContext is not thread-safe).
        // Every per-site command carries the SAME logical
        // deploymentId, so the per-site commands, audit log, persisted record,
        // and UI summary all reference one id instead of N+1 unrelated GUIDs.
        //
        // Each entry also carries the site's unresolved secret references (design §3.3):
        // a non-empty Missing list means this site's command must never reach
        // _communicationService — see the fan-out loop below.
        var siteCommands = new Dictionary<int, (DeployArtifactsCommand Command, IReadOnlyList<SecretReference> Missing)>();
        foreach (var site in sites)
        {
            siteCommands[site.Id] = await BuildDeployArtifactsCommandAsync(
                site.Id, globals, cancellationToken, deploymentId);
        }

        // Deploy to each site in parallel with per-site timeout.
        //
        // Deferred-work register row 60 — what makes this fan-out safe, stated precisely
        // because the invariant is easy to break from a distance:
        //   * `globals` is NOT EF state. FetchGlobalArtifactsAsync projects the entities
        //     into pure Commons message DTOs and drops the entities, so the snapshot the
        //     lambda closes over cannot touch a DbContext at all.
        //   * `site` IS a tracked EF entity, and it is the one piece of EF state that
        //     genuinely crosses into the parallel phase. Only already-materialised scalar
        //     properties are read below (Id, SiteIdentifier, Name) and NO navigation
        //     property is traversed — with lazy loading off (pinned by
        //     ConfigurationDatabase.Tests.AddConfigurationDatabase_DoesNotEnableLazyLoading)
        //     those are plain field reads that never re-enter the shared context.
        // So: do not traverse a navigation property on `site` in here, and do not move
        // per-site command building (which DOES query) inside this lambda — either one
        // puts concurrent load on one non-thread-safe DbContext.
        var tasks = sites.Select(async site =>
        {
            try
            {
                using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
                cts.CancelAfter(_options.ArtifactDeploymentTimeoutPerSite);

                var (command, missing) = siteCommands[site.Id];

                // Fail this site closed BEFORE anything is sent (design §3.3): a referenced
                // secret that is absent or tombstoned at central must not let the rest of the
                // artifact set ship, or the site would read *Current* while a connection
                // cannot resolve its key. Other sites in the batch are unaffected.
                if (missing.Count > 0)
                {
                    var missingMessage = BuildMissingSecretsMessage(missing);
                    _logger.LogError(
                        "Cannot deploy artifacts to site {SiteId} ({SiteName}): {Error}",
                        site.SiteIdentifier, site.Name, missingMessage);
                    return new SiteArtifactResult(site.SiteIdentifier, site.Name, false, missingMessage);
                }

                if (command.Secrets is { Count: > 0 })
                {
                    _logger.LogInformation(
                        "Deploying {Count} secret(s) to site {SiteId}: {Secrets}",
                        command.Secrets.Count, site.SiteIdentifier,
                        string.Join(", ", command.Secrets.Select(s => $"{s.Name}@r{s.Revision}")));
                }

                _logger.LogInformation(
                    "Deploying artifacts to site {SiteId} ({SiteName}), deploymentId={DeploymentId}",
                    site.SiteIdentifier, site.Name, command.DeploymentId);

                var response = await _communicationService.DeployArtifactsAsync(
                    site.SiteIdentifier, command, cts.Token);

                // Task 7 (design §3.4): a reply that only proves receipt is not enough to
                // stamp *Current* — reconcile it against what was actually shipped before
                // this result feeds the stamp decision below.
                return ReconcileSecretsOutcome(command, response, site.SiteIdentifier, site.Name);
            }
            catch (Exception ex) when (ex is TimeoutException or OperationCanceledException or TaskCanceledException)
            {
                _logger.LogWarning(
                    "Artifact deployment to site {SiteId} timed out: {Error}",
                    site.SiteIdentifier, ex.Message);

                return new SiteArtifactResult(
                    site.SiteIdentifier, site.Name, false, $"Timeout: {ex.Message}");
            }
            catch (Exception ex)
            {
                _logger.LogError(ex,
                    "Artifact deployment to site {SiteId} failed",
                    site.SiteIdentifier);

                return new SiteArtifactResult(
                    site.SiteIdentifier, site.Name, false, ex.Message);
            }
        }).ToList();

        var results = await Task.WhenAll(tasks);

        foreach (var result in results)
        {
            perSiteResults[result.SiteId] = result;
        }

        // Task 7 (design §3.4): one best-effort DeploySiteSecrets audit row per site whose
        // command carried or would have carried secrets — covers the reconciled
        // success/failure path AND the fail-closed (missing secret, nothing sent) path, since
        // both already flow into perSiteResults above by the time this loop runs. Driven off
        // siteCommands/perSiteResults rather than inlined into the per-site task above so this
        // is a single serial pass (no concurrent audit writes) that runs — per the ordering
        // rule — BEFORE the best-effort stamp below.
        foreach (var site in sites)
        {
            var (command, missing) = siteCommands[site.Id];
            var result = perSiteResults[site.SiteIdentifier];
            await LogSecretsAuditAsync(
                user, command, missing, site.SiteIdentifier, site.Name,
                result.Success, result.ErrorMessage, cancellationToken);
        }

        // Persist the system artifact deployment record.
        // SystemArtifactDeploymentRecord has no dedicated
        // DeploymentId column (adding one is a Commons/ConfigurationDatabase
        // schema change outside this module). The logical deploymentId is
        // embedded in the PerSiteStatus payload so the persisted record can be
        // correlated with the audit log and UI summary that report the same id.
        var record = new SystemArtifactDeploymentRecord("Artifacts", user)
        {
            DeployedAt = DateTimeOffset.UtcNow,
            PerSiteStatus = JsonSerializer.Serialize(new
            {
                DeploymentId = deploymentId,
                Sites = perSiteResults
            })
        };
        await _deploymentRepo.AddSystemArtifactDeploymentAsync(record, cancellationToken);
        await _deploymentRepo.SaveChangesAsync(cancellationToken);

        // ORDERING RULE: audit row first; the stamp is best-effort bookkeeping and its
        // save failure must not be able to reach the record's save.
        //
        // ISiteRepository and IDeploymentManagerRepository are scoped over the SAME
        // DbContext, so a failed stamp save leaves the Site entries tracked as Modified.
        // Stamping first would therefore let a persistent stamp failure (concurrency
        // conflict, constraint violation) rethrow out of the record's own — uncaught —
        // SaveChangesAsync: the deployment record would be lost and a wholly successful
        // deploy would surface as an exception. Running the record's save first and the
        // stamp after keeps the failure inside StampArtifactRevisionAsync's try/catch,
        // where it is logged at Warning and the deploy still reports Success.
        //
        // Only sites whose deploy SUCCEEDED are stamped. A failed site is deliberately
        // left untouched so it keeps its previous stamp (or none at all) and keeps
        // reading "out of date" on the Sites page.
        var successfulSites = sites
            .Where(site => perSiteResults.TryGetValue(site.SiteIdentifier, out var r) && r.Success)
            .ToList();
        if (successfulSites.Count > 0)
        {
            await StampArtifactRevisionAsync(
                successfulSites.Select(site => (site, siteCommands[site.Id].Command)).ToList(),
                cancellationToken);
        }

        var summary = new ArtifactDeploymentSummary(
            deploymentId,
            results.ToList(),
            results.Count(r => r.Success),
            results.Count(r => !r.Success));

        await _auditService.LogAsync(user, "DeployArtifacts", "SystemArtifact",
            deploymentId, "Artifacts",
            new { summary.SuccessCount, summary.FailureCount },
            cancellationToken);

        return Result<ArtifactDeploymentSummary>.Success(summary);
    }

    /// <summary>
    /// Retry artifact deployment to a specific site that previously failed.
    /// </summary>
    /// <param name="siteDbId">The database identifier of the site.</param>
    /// <param name="siteIdentifier">The site identifier string.</param>
    /// <param name="user">The user initiating the retry.</param>
    /// <param name="cancellationToken">Cancellation token.</param>
    /// <returns>The result of the retry operation for the site.</returns>
    public async Task<Result<SiteArtifactResult>> RetryForSiteAsync(
        int siteDbId,
        string siteIdentifier,
        string user,
        CancellationToken cancellationToken = default)
    {
        try
        {
            using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            cts.CancelAfter(_options.ArtifactDeploymentTimeoutPerSite);

            var (command, missing) = await BuildDeployArtifactsCommandAsync(siteDbId, cts.Token);

            // Same fail-closed rule as the sweep path (design §3.3): a retry must not ship a
            // site an artifact set whose secret references still don't resolve.
            if (missing.Count > 0)
            {
                var missingMessage = BuildMissingSecretsMessage(missing);
                _logger.LogError(
                    "Cannot deploy artifacts to site {SiteId}: {Error}", siteIdentifier, missingMessage);
                // Task 7: the fail-closed case still gets its DeploySiteSecrets row — nothing
                // was sent, but the command carried (or would have carried) secrets.
                await LogSecretsAuditAsync(
                    user, command, missing, siteIdentifier, siteIdentifier, false, missingMessage,
                    cancellationToken);
                return Result<SiteArtifactResult>.Failure(missingMessage);
            }

            if (command.Secrets is { Count: > 0 })
            {
                _logger.LogInformation(
                    "Deploying {Count} secret(s) to site {SiteId}: {Secrets}",
                    command.Secrets.Count, siteIdentifier,
                    string.Join(", ", command.Secrets.Select(s => $"{s.Name}@r{s.Revision}")));
            }

            var response = await _communicationService.DeployArtifactsAsync(siteIdentifier, command, cts.Token);

            // Task 7 (design §3.4): reconcile the raw wire response against what was
            // actually shipped before it feeds the audit row, the stamp decision, or the
            // returned Result — an old site's/partial site's reply must not read as success.
            var result = ReconcileSecretsOutcome(command, response, siteIdentifier, siteIdentifier);

            await _auditService.LogAsync(user, "RetryArtifactDeployment", "SystemArtifact",
                command.DeploymentId, siteIdentifier, new { result.Success }, cancellationToken);

            // Ordering rule: audit before the best-effort stamp.
            await LogSecretsAuditAsync(
                user, command, missing, siteIdentifier, siteIdentifier,
                result.Success, result.ErrorMessage, cancellationToken);

            // Same stamp-on-success rule as the sweep path: a retry that lands is a
            // successful artifact deploy and must move the site's revision stamp; a
            // retry that fails leaves the previous stamp in place.
            //
            // Same ORDERING RULE too — audit row first, best-effort stamp after. Here the
            // method's own catch would convert a stamp exception into a Failure result for
            // an artifact deploy that actually succeeded, so the stamp must (and does) keep
            // its own try/catch inside StampArtifactRevisionAsync rather than lean on this
            // one.
            if (result.Success)
            {
                await StampRetriedSiteAsync(siteDbId, command, cancellationToken);
            }

            return result.Success
                ? Result<SiteArtifactResult>.Success(result)
                : Result<SiteArtifactResult>.Failure(result.ErrorMessage ?? "Retry failed.");
        }
        catch (Exception ex)
        {
            return Result<SiteArtifactResult>.Failure($"Retry failed for site {siteIdentifier}: {ex.Message}");
        }
    }

    /// <summary>
    /// Computes what each given site's artifact revision hash WOULD be if the current
    /// artifact definitions were deployed right now. Comparing the result against the
    /// site's stored <see cref="Site.ArtifactRevisionHash"/> is how the pending-updates
    /// aggregate decides whether a site's system-wide artifacts are out of date.
    /// </summary>
    /// <param name="sites">The sites to compute current hashes for.</param>
    /// <param name="cancellationToken">Cancellation token.</param>
    /// <returns>
    /// A map of site database id to the hash of the artifact set that a deploy would send
    /// to that site right now.
    /// </returns>
    /// <remarks>
    /// The global artifact queries run ONCE for the whole sweep (the same hoist
    /// <see cref="DeployToAllSitesAsync"/> performs); only the per-site data-connection query
    /// runs per site. The commands are built with the constant deployment id <c>"current"</c>,
    /// which is harmless because <see cref="ArtifactRevisionHashService"/> excludes both the
    /// deployment id and the command timestamp from the hash — that exclusion is what makes a
    /// computed-now hash comparable with one stamped by a past deploy.
    /// <para>
    /// A failing per-site data-connection query aborts the WHOLE batch by design: a partial map
    /// would silently report unqueried sites as up to date. The pending-updates cache discards a
    /// faulted flight and the UI shows the aggregate as unavailable instead.
    /// </para>
    /// </remarks>
    public async Task<IReadOnlyDictionary<int, string>> ComputeCurrentArtifactHashesAsync(
        IReadOnlyList<Site> sites,
        CancellationToken cancellationToken = default)
    {
        ArgumentNullException.ThrowIfNull(sites);

        var hashes = new Dictionary<int, string>();
        if (sites.Count == 0)
            return hashes;

        var globals = await FetchGlobalArtifactsAsync(cancellationToken);
        foreach (var site in sites)
        {
            // The Missing tuple element is deliberately unused here: ComputeHash reads
            // MissingSecrets straight off the command (design §3.4), so an unresolved
            // reference still folds a distinguishable marker into the hash — the site reads
            // *Out of date* instead of this call failing the whole batch.
            var (command, _) = await BuildDeployArtifactsCommandAsync(
                site.Id, globals, cancellationToken, deploymentId: "current");
            hashes[site.Id] = _hashService.ComputeHash(command);
        }

        return hashes;
    }

    /// <summary>
    /// Loads the retried site and stamps the artifact revision it just accepted. Best-effort
    /// end to end: the load is caught here and the stamp carries its own catch, so nothing on
    /// this path can escape into <see cref="RetryForSiteAsync"/>'s catch and report a
    /// successful artifact deploy as a failure.
    /// </summary>
    private async Task StampRetriedSiteAsync(
        int siteDbId,
        DeployArtifactsCommand command,
        CancellationToken cancellationToken)
    {
        Site? site;
        try
        {
            site = await _siteRepo.GetSiteByIdAsync(siteDbId, cancellationToken);
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex,
                "Artifact retry for site {SiteDbId} succeeded but the site row could not be loaded to " +
                "stamp its artifact revision; it will report out-of-date artifacts until the next deploy",
                siteDbId);
            return;
        }

        if (site is null)
            return;

        await StampArtifactRevisionAsync([(site, command)], cancellationToken);
    }

    /// <summary>
    /// Stamps <see cref="Site.ArtifactRevisionHash"/> and <see cref="Site.ArtifactsDeployedAt"/>
    /// for every (site, deployed command) pair and flushes them in one save. Best-effort: a
    /// failure is logged at Warning and swallowed, because the artifacts DID reach the sites —
    /// losing the stamp only means the Sites page reports those sites out of date until the
    /// next deploy, which is strictly better than reporting a successful deploy as failed.
    /// </summary>
    private async Task StampArtifactRevisionAsync(
        IReadOnlyList<(Site Site, DeployArtifactsCommand Command)> deployed,
        CancellationToken cancellationToken)
    {
        try
        {
            var stampedAt = DateTimeOffset.UtcNow;
            foreach (var (site, command) in deployed)
            {
                site.ArtifactRevisionHash = _hashService.ComputeHash(command);
                site.ArtifactsDeployedAt = stampedAt;
                await _siteRepo.UpdateSiteAsync(site, cancellationToken);
            }

            await _siteRepo.SaveChangesAsync(cancellationToken);
        }
        catch (Exception ex)
        {
            _logger.LogWarning(ex,
                "Artifact deployment succeeded but stamping the artifact revision failed for site(s) {Sites}; " +
                "they will report out-of-date artifacts until the next deploy",
                string.Join(", ", deployed.Select(d => d.Site.SiteIdentifier)));
        }
    }
}

/// <summary>
/// Per-site result for artifact deployment.
/// </summary>
/// <param name="SecretsDeployed">
/// How many secrets the site confirmed applying, when the deployed command shipped at
/// least one (task 7, design §3.4). <c>null</c> when the site shipped no secrets at all
/// — deliberately the same value an old site's unset <see cref="ArtifactDeploymentResponse.SecretsApplied"/>
/// produces, so the two "no secrets involved" cases (nothing shipped vs. an old site that
/// doesn't know about the field) are indistinguishable, which is correct: neither one is a
/// secrets-gating concern.
/// </param>
public record SiteArtifactResult(
    string SiteId,
    string SiteName,
    bool Success,
    string? ErrorMessage,
    int? SecretsDeployed = null);

/// <summary>
/// Summary of system-wide artifact deployment with per-site results.
/// </summary>
public record ArtifactDeploymentSummary(
    string DeploymentId,
    IReadOnlyList<SiteArtifactResult> SiteResults,
    int SuccessCount,
    int FailureCount);