ScadaBridge/src/ZB.MOM.WW.ScadaBridge.DeploymentManager/ArtifactDeploymentService.cs

using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Communication;

namespace ZB.MOM.WW.ScadaBridge.DeploymentManager;

/// <summary>
/// System-wide artifact deployment.
/// Broadcasts artifacts (shared scripts, external systems, DB connections, and
/// data connections) to all sites with per-site tracking.
///
/// Notification lists and SMTP configuration are deliberately NOT shipped to
/// sites: notification delivery is central-only (sites store-and-forward to
/// central and never talk to SMTP), so no notification artifact or SMTP
/// credential is ever distributed to a site. The
/// <see cref="DeployArtifactsCommand"/> still carries the
/// <c>NotificationLists</c>/<c>SmtpConfigurations</c> fields for additive
/// message-contract compatibility, but central never populates them.
///
/// - Successful sites are NOT rolled back on other failures.
/// - Failed sites are retryable individually.
/// - 120s timeout per site.
/// - Cross-site version skew is supported.
/// </summary>
public class ArtifactDeploymentService
{
    private readonly ISiteRepository _siteRepo;
    private readonly IDeploymentManagerRepository _deploymentRepo;
    private readonly ITemplateEngineRepository _templateRepo;
    private readonly IExternalSystemRepository _externalSystemRepo;
    private readonly CommunicationService _communicationService;
    private readonly IAuditService _auditService;
    private readonly DeploymentManagerOptions _options;
    private readonly ILogger<ArtifactDeploymentService> _logger;

    /// <summary>
    /// Initializes a new instance of the ArtifactDeploymentService.
    /// </summary>
    /// <param name="siteRepo">Repository for site data.</param>
    /// <param name="deploymentRepo">Repository for deployment records.</param>
    /// <param name="templateRepo">Repository for templates.</param>
    /// <param name="externalSystemRepo">Repository for external systems.</param>
    /// <param name="notificationRepo">
    /// Retained on the signature for DI/source compatibility but
    /// intentionally NOT consumed. Notification lists and SMTP configuration are
    /// central-only and are never shipped to sites, so the artifact path must not read
    /// the notification repository at all.
    /// </param>
    /// <param name="communicationService">Service for communicating with sites.</param>
    /// <param name="auditService">Service for audit logging.</param>
    /// <param name="options">Deployment manager options.</param>
    /// <param name="logger">Logger instance.</param>
    public ArtifactDeploymentService(
        ISiteRepository siteRepo,
        IDeploymentManagerRepository deploymentRepo,
        ITemplateEngineRepository templateRepo,
        IExternalSystemRepository externalSystemRepo,
        INotificationRepository notificationRepo,
        CommunicationService communicationService,
        IAuditService auditService,
        IOptions<DeploymentManagerOptions> options,
        ILogger<ArtifactDeploymentService> logger)
    {
        _siteRepo = siteRepo;
        _deploymentRepo = deploymentRepo;
        _templateRepo = templateRepo;
        _externalSystemRepo = externalSystemRepo;
        // notificationRepo is deliberately not stored — notification
        // lists and SMTP configs are central-only and are never fetched for shipping to sites.
        _ = notificationRepo;
        _communicationService = communicationService;
        _auditService = auditService;
        _options = options.Value;
        _logger = logger;
    }

    /// <summary>
    /// Collects all artifact types from repositories and builds a <see cref="DeployArtifactsCommand"/>
    /// scoped to a specific site's data connections.
    /// </summary>
    /// <param name="siteId">The DB id of the site whose data connections are collected.</param>
    /// <param name="cancellationToken">Cancellation token.</param>
    /// <param name="deploymentId">
    /// The logical deployment id for this artifact deployment. All per-site
    /// commands of one <see cref="DeployToAllSitesAsync"/> call share this id so the audit log,
    /// UI summary, and persisted record correlate. When <c>null</c> a fresh id is minted (used by
    /// single-site retries).
    /// </param>
    /// <returns>A deployment artifacts command for the site.</returns>
    /// <remarks>
    /// This convenience overload runs the global artifact queries
    /// for a single site (used by <see cref="RetryForSiteAsync"/>). The multi-site
    /// <see cref="DeployToAllSitesAsync"/> path hoists the global queries OUT of the
    /// per-site loop and calls the prefetched-globals overload to avoid the N+1
    /// re-query of every system-wide artifact set per site.
    /// </remarks>
    public async Task<DeployArtifactsCommand> BuildDeployArtifactsCommandAsync(
        int siteId,
        CancellationToken cancellationToken = default,
        string? deploymentId = null)
    {
        var globals = await FetchGlobalArtifactsAsync(cancellationToken);
        return await BuildDeployArtifactsCommandAsync(siteId, globals, cancellationToken, deploymentId);
    }

    /// <summary>
    /// Builds a per-site <see cref="DeployArtifactsCommand"/> using a previously-fetched
    /// snapshot of the global artifact sets (shared scripts, external systems + methods,
    /// DB connections). Only the per-site data-connection query runs here.
    /// </summary>
    /// <remarks>
    /// Separating the global fetch from the per-site build lets
    /// <see cref="DeployToAllSitesAsync"/> issue the global queries exactly once across
    /// the whole multi-site sweep, eliminating the N+1 re-query of shared scripts,
    /// external systems, methods, and DB connections.
    ///
    /// The command's <c>NotificationLists</c> and
    /// <c>SmtpConfigurations</c> fields are always sent <c>null</c> — notification
    /// delivery is central-only and no notification artifact or SMTP credential is
    /// ever distributed to a site. The fields remain on the contract only for
    /// additive compatibility.
    /// </remarks>
    private async Task<DeployArtifactsCommand> BuildDeployArtifactsCommandAsync(
        int siteId,
        GlobalArtifactSnapshot globals,
        CancellationToken cancellationToken,
        string? deploymentId)
    {
        var dataConnections = await _siteRepo.GetDataConnectionsBySiteIdAsync(siteId, cancellationToken);

        // Map data connections
        var dataConnectionArtifacts = dataConnections.Select(dc =>
            new DataConnectionArtifact(dc.Name, dc.Protocol, dc.PrimaryConfiguration, dc.BackupConfiguration, dc.FailoverRetryCount)).ToList();

        return new DeployArtifactsCommand(
            deploymentId ?? Guid.NewGuid().ToString("N"),
            globals.SharedScripts,
            globals.ExternalSystems,
            globals.DatabaseConnections,
            // Notification lists are central-only — never shipped to sites.
            NotificationLists: null,
            dataConnectionArtifacts,
            // SMTP config (incl. credentials) is central-only — never shipped to sites.
            SmtpConfigurations: null,
            DateTimeOffset.UtcNow);
    }

    /// <summary>
    /// Fetches the system-wide artifact sets that are identical across every site —
    /// shared scripts, external systems (with their methods serialized in), and
    /// database connections. Used by <see cref="DeployToAllSitesAsync"/> to pre-load
    /// once before the per-site loop.
    /// </summary>
    /// <remarks>
    /// The per-site artifact build path previously re-issued
    /// every one of these queries per site (≈ N + M·N round trips for N sites
    /// and M external systems). Hoisting them here drops that to a single fetch.
    ///
    /// Notification lists and SMTP configurations are NOT
    /// fetched here. Notification delivery is central-only, so they are never
    /// shipped to sites — the artifact path must not even read them.
    /// </remarks>
    private async Task<GlobalArtifactSnapshot> FetchGlobalArtifactsAsync(CancellationToken cancellationToken)
    {
        var sharedScripts = await _templateRepo.GetAllSharedScriptsAsync(cancellationToken);
        var externalSystems = await _externalSystemRepo.GetAllExternalSystemsAsync(cancellationToken);
        var dbConnections = await _externalSystemRepo.GetAllDatabaseConnectionsAsync(cancellationToken);

        // Map shared scripts
        var scriptArtifacts = sharedScripts.Select(s =>
            new SharedScriptArtifact(s.Name, s.Code, s.ParameterDefinitions, s.ReturnDefinition)).ToList();

        // Map external systems (serialize methods per system)
        var externalSystemArtifacts = new List<ExternalSystemArtifact>();
        foreach (var es in externalSystems)
        {
            var methods = await _externalSystemRepo.GetMethodsByExternalSystemIdAsync(es.Id, cancellationToken);
            var methodsJson = methods.Count > 0
                ? JsonSerializer.Serialize(methods.Select(m => new
                {
                    m.Name,
                    m.HttpMethod,
                    m.Path,
                    m.ParameterDefinitions,
                    m.ReturnDefinition
                }))
                : null;
            // MaxRetries/RetryDelay ride along so the site's Store-and-Forward buffering honours
            // the centrally-configured per-system retry policy. Before they were carried, a site
            // silently fell back to the engine defaults (50 x 30 s) for every cached call, whatever
            // central had configured — the DatabaseConnectionArtifact below always carried them.
            externalSystemArtifacts.Add(new ExternalSystemArtifact(
                es.Name, es.EndpointUrl, es.AuthType, es.AuthConfiguration, methodsJson, es.TimeoutSeconds,
                es.MaxRetries, es.RetryDelay));
        }

        // Map database connections
        var dbConnectionArtifacts = dbConnections.Select(d =>
            new DatabaseConnectionArtifact(d.Name, d.ConnectionString, d.MaxRetries, d.RetryDelay)).ToList();

        return new GlobalArtifactSnapshot(
            scriptArtifacts,
            externalSystemArtifacts,
            dbConnectionArtifacts);
    }

    /// <summary>
    /// Bag of the global artifact sets that do not vary per site, captured once at
    /// the start of <see cref="DeployToAllSitesAsync"/> and reused for every per-site
    /// command build. Notification lists and SMTP
    /// configurations are deliberately absent — they are central-only and never
    /// shipped to sites.
    /// </summary>
    private sealed record GlobalArtifactSnapshot(
        IReadOnlyList<SharedScriptArtifact> SharedScripts,
        IReadOnlyList<ExternalSystemArtifact> ExternalSystems,
        IReadOnlyList<DatabaseConnectionArtifact> DatabaseConnections);

    /// <summary>
    /// Deploys artifacts to all sites. Builds a per-site command with that site's data connections.
    /// Returns per-site result matrix.
    /// </summary>
    /// <param name="user">The user initiating the deployment.</param>
    /// <param name="cancellationToken">Cancellation token.</param>
    /// <returns>A summary of the artifact deployment results for all sites.</returns>
    public virtual async Task<Result<ArtifactDeploymentSummary>> DeployToAllSitesAsync(
        string user,
        CancellationToken cancellationToken = default)
    {
        var sites = await _siteRepo.GetAllSitesAsync(cancellationToken);
        if (sites.Count == 0)
            return Result<ArtifactDeploymentSummary>.Failure("No sites configured.");

        return await DeployCoreAsync(sites, user, cancellationToken);
    }

    /// <summary>
    /// Deploys system-wide artifacts to a single site (management-API --site-id path).
    /// </summary>
    /// <param name="siteId">The database identifier of the target site.</param>
    /// <param name="user">The user initiating the deployment.</param>
    /// <param name="cancellationToken">Cancellation token.</param>
    /// <returns>A summary of the artifact deployment result for the single site.</returns>
    public virtual async Task<Result<ArtifactDeploymentSummary>> DeployToSiteAsync(
        int siteId,
        string user,
        CancellationToken cancellationToken = default)
    {
        var site = await _siteRepo.GetSiteByIdAsync(siteId, cancellationToken);
        if (site is null)
            return Result<ArtifactDeploymentSummary>.Failure($"Site with ID {siteId} not found.");

        return await DeployCoreAsync([site], user, cancellationToken);
    }

    /// <summary>
    /// Core deployment path shared by the all-sites and single-site entry points.
    /// Builds a per-site command with that site's data connections and deploys to
    /// every supplied site under one logical deployment id.
    /// </summary>
    private async Task<Result<ArtifactDeploymentSummary>> DeployCoreAsync(
        IReadOnlyList<Site> sites,
        string user,
        CancellationToken cancellationToken)
    {
        var deploymentId = Guid.NewGuid().ToString("N");
        var perSiteResults = new Dictionary<string, SiteArtifactResult>();

        // Hoist the system-wide artifact queries (shared scripts,
        // external systems + methods, DB connections) OUT of the per-site loop so they
        // run ONCE instead of once per site. Only data connections legitimately vary
        // per site, so they stay inside the loop. (Notification lists and SMTP config
        // are central-only and not fetched at all.)
        var globals = await FetchGlobalArtifactsAsync(cancellationToken);

        // Build per-site commands sequentially (DbContext is not thread-safe).
        // Every per-site command carries the SAME logical
        // deploymentId, so the per-site commands, audit log, persisted record,
        // and UI summary all reference one id instead of N+1 unrelated GUIDs.
        var siteCommands = new Dictionary<int, DeployArtifactsCommand>();
        foreach (var site in sites)
        {
            siteCommands[site.Id] = await BuildDeployArtifactsCommandAsync(
                site.Id, globals, cancellationToken, deploymentId);
        }

        // Deploy to each site in parallel with per-site timeout.
        //
        // Deferred-work register row 60 — what makes this fan-out safe, stated precisely
        // because the invariant is easy to break from a distance:
        //   * `globals` is NOT EF state. FetchGlobalArtifactsAsync projects the entities
        //     into pure Commons message DTOs and drops the entities, so the snapshot the
        //     lambda closes over cannot touch a DbContext at all.
        //   * `site` IS a tracked EF entity, and it is the one piece of EF state that
        //     genuinely crosses into the parallel phase. Only already-materialised scalar
        //     properties are read below (Id, SiteIdentifier, Name) and NO navigation
        //     property is traversed — with lazy loading off (pinned by
        //     ConfigurationDatabase.Tests.AddConfigurationDatabase_DoesNotEnableLazyLoading)
        //     those are plain field reads that never re-enter the shared context.
        // So: do not traverse a navigation property on `site` in here, and do not move
        // per-site command building (which DOES query) inside this lambda — either one
        // puts concurrent load on one non-thread-safe DbContext.
        var tasks = sites.Select(async site =>
        {
            try
            {
                using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
                cts.CancelAfter(_options.ArtifactDeploymentTimeoutPerSite);

                var command = siteCommands[site.Id];

                _logger.LogInformation(
                    "Deploying artifacts to site {SiteId} ({SiteName}), deploymentId={DeploymentId}",
                    site.SiteIdentifier, site.Name, command.DeploymentId);

                var response = await _communicationService.DeployArtifactsAsync(
                    site.SiteIdentifier, command, cts.Token);

                return new SiteArtifactResult(
                    site.SiteIdentifier, site.Name, response.Success, response.ErrorMessage);
            }
            catch (Exception ex) when (ex is TimeoutException or OperationCanceledException or TaskCanceledException)
            {
                _logger.LogWarning(
                    "Artifact deployment to site {SiteId} timed out: {Error}",
                    site.SiteIdentifier, ex.Message);

                return new SiteArtifactResult(
                    site.SiteIdentifier, site.Name, false, $"Timeout: {ex.Message}");
            }
            catch (Exception ex)
            {
                _logger.LogError(ex,
                    "Artifact deployment to site {SiteId} failed",
                    site.SiteIdentifier);

                return new SiteArtifactResult(
                    site.SiteIdentifier, site.Name, false, ex.Message);
            }
        }).ToList();

        var results = await Task.WhenAll(tasks);

        foreach (var result in results)
        {
            perSiteResults[result.SiteId] = result;
        }

        // Persist the system artifact deployment record.
        // SystemArtifactDeploymentRecord has no dedicated
        // DeploymentId column (adding one is a Commons/ConfigurationDatabase
        // schema change outside this module). The logical deploymentId is
        // embedded in the PerSiteStatus payload so the persisted record can be
        // correlated with the audit log and UI summary that report the same id.
        var record = new SystemArtifactDeploymentRecord("Artifacts", user)
        {
            DeployedAt = DateTimeOffset.UtcNow,
            PerSiteStatus = JsonSerializer.Serialize(new
            {
                DeploymentId = deploymentId,
                Sites = perSiteResults
            })
        };
        await _deploymentRepo.AddSystemArtifactDeploymentAsync(record, cancellationToken);
        await _deploymentRepo.SaveChangesAsync(cancellationToken);

        var summary = new ArtifactDeploymentSummary(
            deploymentId,
            results.ToList(),
            results.Count(r => r.Success),
            results.Count(r => !r.Success));

        await _auditService.LogAsync(user, "DeployArtifacts", "SystemArtifact",
            deploymentId, "Artifacts",
            new { summary.SuccessCount, summary.FailureCount },
            cancellationToken);

        return Result<ArtifactDeploymentSummary>.Success(summary);
    }

    /// <summary>
    /// Retry artifact deployment to a specific site that previously failed.
    /// </summary>
    /// <param name="siteDbId">The database identifier of the site.</param>
    /// <param name="siteIdentifier">The site identifier string.</param>
    /// <param name="user">The user initiating the retry.</param>
    /// <param name="cancellationToken">Cancellation token.</param>
    /// <returns>The result of the retry operation for the site.</returns>
    public async Task<Result<SiteArtifactResult>> RetryForSiteAsync(
        int siteDbId,
        string siteIdentifier,
        string user,
        CancellationToken cancellationToken = default)
    {
        try
        {
            using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            cts.CancelAfter(_options.ArtifactDeploymentTimeoutPerSite);

            var command = await BuildDeployArtifactsCommandAsync(siteDbId, cts.Token);
            var response = await _communicationService.DeployArtifactsAsync(siteIdentifier, command, cts.Token);

            var result = new SiteArtifactResult(siteIdentifier, siteIdentifier, response.Success, response.ErrorMessage);

            await _auditService.LogAsync(user, "RetryArtifactDeployment", "SystemArtifact",
                command.DeploymentId, siteIdentifier, new { response.Success }, cancellationToken);

            return response.Success
                ? Result<SiteArtifactResult>.Success(result)
                : Result<SiteArtifactResult>.Failure(response.ErrorMessage ?? "Retry failed.");
        }
        catch (Exception ex)
        {
            return Result<SiteArtifactResult>.Failure($"Retry failed for site {siteIdentifier}: {ex.Message}");
        }
    }
}

/// <summary>
/// Per-site result for artifact deployment.
/// </summary>
public record SiteArtifactResult(
    string SiteId,
    string SiteName,
    bool Success,
    string? ErrorMessage);

/// <summary>
/// Summary of system-wide artifact deployment with per-site results.
/// </summary>
public record ArtifactDeploymentSummary(
    string DeploymentId,
    IReadOnlyList<SiteArtifactResult> SiteResults,
    int SuccessCount,
    int FailureCount);