feat(deploy): PendingUpdatesEvaluator — one fleet snapshot of stale instances and stale-artifact sites

One scoped read that answers "is anything waiting to be deployed?" for the whole
fleet: each site's stored artifact revision hash compared against what a deploy
would send right now (NeverDeployed / Current / OutOfDate), plus per-site
deployed/stale/unknown instance tallies from the batched staleness evaluation.
NeverDeployed is deliberately NOT a pending update — the operator has not deployed
yet — and exceptions propagate so the caller's single-flight cache can render
"unavailable" rather than invent an "up to date". Adds the validated
PendingUpdatesCacheTtl (60 s) that TTL will key off.

Claude-Session: https://claude.ai/code/session_01Y5ef34iUUMjmwnMpL9xgmB

src/ZB.MOM.WW.ScadaBridge.DeploymentManager/DeploymentManagerOptions.cs

@@ -75,4 +75,19 @@ public class DeploymentManagerOptions
     /// </para>
     /// </summary>
     public TimeSpan TerminalDeploymentRecordRetention { get; set; } = TimeSpan.FromDays(365);
+
+    /// <summary>
+    /// How long the Central UI pending-updates snapshot stays fresh; one fleet
+    /// flatten per TTL while any Deployer/Administrator circuit is open.
+    ///
+    /// <para>
+    /// The snapshot is a whole-fleet read (every site's artifact hash plus every
+    /// deployed instance's staleness), so it is held behind a single-flight TTL
+    /// cache: however many circuits poll, and however often, the fleet is evaluated
+    /// at most once per this window — and not at all when nobody is looking. Must be
+    /// positive; zero would make every poll a cache miss and re-flatten the fleet per
+    /// circuit, which is precisely the cost the cache exists to avoid.
+    /// </para>
+    /// </summary>
+    public TimeSpan PendingUpdatesCacheTtl { get; set; } = TimeSpan.FromSeconds(60);
 }

src/ZB.MOM.WW.ScadaBridge.DeploymentManager/DeploymentManagerOptionsValidator.cs

@@ -43,6 +43,12 @@ public sealed class DeploymentManagerOptionsValidator : OptionsValidatorBase<Dep
             $"(was {options.SiteDeploymentReconcileTimeout}); it bounds each query-before-redeploy " +
             $"reconciliation inside a bulk site deployment's serial prepare phase.");
 
+        builder.RequireThat(options.PendingUpdatesCacheTtl > TimeSpan.Zero,
+            $"ScadaBridge:DeploymentManager:PendingUpdatesCacheTtl must be a positive duration " +
+            $"(was {options.PendingUpdatesCacheTtl}); it is the freshness window of the Central UI " +
+            $"pending-updates snapshot, and a non-positive value would re-flatten the whole fleet on " +
+            $"every poll of every open circuit.");
+
         builder.RequireThat(options.TerminalDeploymentRecordRetention >= TimeSpan.Zero,
             $"ScadaBridge:DeploymentManager:TerminalDeploymentRecordRetention must not be negative " +
             $"(was {options.TerminalDeploymentRecordRetention}); use TimeSpan.Zero to disable purging of " +

src/ZB.MOM.WW.ScadaBridge.DeploymentManager/PendingUpdates/IPendingUpdatesEvaluator.cs

@@ -0,0 +1,24 @@
+namespace ZB.MOM.WW.ScadaBridge.DeploymentManager.PendingUpdates;
+
+/// <summary>
+/// Computes the fleet-wide "pending updates" aggregate — which sites are running
+/// out-of-date system-wide artifacts and how many deployed instances have drifted
+/// from their deployed configuration.
+///
+/// <para>
+/// Scoped, and deliberately uncached at this seam: caching belongs to the Central
+/// UI's single-flight TTL cache, which runs this evaluator in its own DI scope so
+/// the fleet is flattened at most once per TTL regardless of how many circuits are
+/// watching. Callers that resolve the evaluator directly pay a full fleet flatten
+/// per call.
+/// </para>
+/// </summary>
+public interface IPendingUpdatesEvaluator
+{
+    /// <summary>
+    /// Evaluates every site and every deployed instance, returning one snapshot.
+    /// </summary>
+    /// <param name="cancellationToken">Cancellation token, honoured between instances.</param>
+    /// <returns>A snapshot of the fleet's pending updates as of now.</returns>
+    Task<PendingUpdatesSnapshot> EvaluateAsync(CancellationToken cancellationToken = default);
+}

src/ZB.MOM.WW.ScadaBridge.DeploymentManager/PendingUpdates/PendingUpdatesEvaluator.cs

@@ -0,0 +1,172 @@
+using System.Diagnostics;
+using Microsoft.Extensions.Logging;
+using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
+using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
+
+namespace ZB.MOM.WW.ScadaBridge.DeploymentManager.PendingUpdates;
+
+/// <summary>
+/// Default <see cref="IPendingUpdatesEvaluator"/>: one pass over the fleet that
+/// answers both halves of "is anything waiting to be deployed?" — per-site artifact
+/// staleness and per-instance configuration staleness.
+///
+/// <para>
+/// <b>How it stays affordable.</b> Both halves are batch reads, not per-entity loops.
+/// <see cref="ArtifactDeploymentService.ComputeCurrentArtifactHashesAsync"/> hoists
+/// the global artifact queries once for the whole fleet, and
+/// <see cref="DeploymentService.GetInstanceStalenessAsync"/> runs every flatten
+/// through ONE <see cref="FlattenSession"/> so a template chain shared by N
+/// instances is walked once. It is still a whole-fleet flatten, which is why the
+/// caller caches it behind a TTL rather than calling it per request.
+/// </para>
+///
+/// <para>
+/// <b>Exceptions propagate.</b> Nothing is swallowed and no partial snapshot is
+/// invented: the caller's single-flight cache discards a faulted flight and the UI
+/// renders <i>unavailable</i>. Degrading a failed read to "up to date" is the one
+/// answer that must never be manufactured — it would silently hide a fleet waiting
+/// on a deploy.
+/// </para>
+/// </summary>
+public sealed class PendingUpdatesEvaluator : IPendingUpdatesEvaluator
+{
+    private readonly ISiteRepository _siteRepository;
+    private readonly ITemplateEngineRepository _templateRepository;
+    private readonly ArtifactDeploymentService _artifactDeploymentService;
+    private readonly DeploymentService _deploymentService;
+    private readonly ILogger<PendingUpdatesEvaluator> _logger;
+
+    /// <summary>
+    /// Initializes a new evaluator.
+    /// </summary>
+    /// <param name="siteRepository">Repository the site list is read from.</param>
+    /// <param name="templateRepository">Repository the instance list is read from.</param>
+    /// <param name="artifactDeploymentService">Computes each site's current artifact revision hash.</param>
+    /// <param name="deploymentService">Evaluates instance staleness in one batched flatten session.</param>
+    /// <param name="logger">Logger instance.</param>
+    public PendingUpdatesEvaluator(
+        ISiteRepository siteRepository,
+        ITemplateEngineRepository templateRepository,
+        ArtifactDeploymentService artifactDeploymentService,
+        DeploymentService deploymentService,
+        ILogger<PendingUpdatesEvaluator> logger)
+    {
+        _siteRepository = siteRepository;
+        _templateRepository = templateRepository;
+        _artifactDeploymentService = artifactDeploymentService;
+        _deploymentService = deploymentService;
+        _logger = logger;
+    }
+
+    /// <inheritdoc />
+    public async Task<PendingUpdatesSnapshot> EvaluateAsync(CancellationToken cancellationToken = default)
+    {
+        var stopwatch = Stopwatch.StartNew();
+
+        var sites = await _siteRepository.GetAllSitesAsync(cancellationToken);
+        var instances = await _templateRepository.GetAllInstancesAsync(cancellationToken);
+
+        var currentArtifactHashes =
+            await _artifactDeploymentService.ComputeCurrentArtifactHashesAsync(sites, cancellationToken);
+
+        // Only instances the operator has actually deployed can be stale; a
+        // NotDeployed instance has nothing to compare against, and flattening it
+        // would be pure cost on every TTL expiry.
+        var deployedInstances = instances.Where(i => i.State != InstanceState.NotDeployed).ToList();
+        var staleness = await _deploymentService.GetInstanceStalenessAsync(
+            deployedInstances.Select(i => i.Id).ToList(), cancellationToken);
+
+        var rows = new List<SitePendingUpdates>(sites.Count);
+        foreach (var site in sites.OrderBy(s => s.Name, StringComparer.OrdinalIgnoreCase))
+        {
+            var (deployed, stale, unknown) = TallyInstances(site.Id, deployedInstances, staleness);
+            rows.Add(new SitePendingUpdates(
+                site.Id,
+                site.SiteIdentifier,
+                site.Name,
+                ResolveArtifactState(site, currentArtifactHashes),
+                site.ArtifactsDeployedAt,
+                deployed,
+                stale,
+                unknown));
+        }
+
+        var snapshot = new PendingUpdatesSnapshot(DateTimeOffset.UtcNow, rows);
+
+        // One line per evaluation — this runs at most once per cache TTL, so it is
+        // the cheapest record of what the fleet looked like and what the sweep cost.
+        _logger.LogInformation(
+            "Pending-updates snapshot: {SiteCount} site(s), {DeployedInstanceCount} deployed instance(s), " +
+            "{StaleInstanceCount} stale instance(s), {StaleArtifactSiteCount} site(s) with out-of-date " +
+            "artifacts, in {ElapsedMs} ms",
+            rows.Count,
+            rows.Sum(r => r.DeployedInstanceCount),
+            snapshot.StaleInstanceCount,
+            snapshot.StaleArtifactSiteCount,
+            stopwatch.ElapsedMilliseconds);
+
+        return snapshot;
+    }
+
+    /// <summary>
+    /// Decides a site's artifact state by comparing the hash stamped by its last
+    /// successful artifact deploy against the hash the current definitions would
+    /// produce. A site with no stamp has never had artifacts deployed, which is
+    /// <see cref="ArtifactDeploymentState.NeverDeployed"/>, NOT out of date. A site
+    /// missing from the computed map (it appeared between the two reads) is reported
+    /// out of date, the side that prompts an operator to look rather than hiding it.
+    /// </summary>
+    /// <param name="site">The site to classify.</param>
+    /// <param name="currentArtifactHashes">Site id to the hash a deploy would send right now.</param>
+    /// <returns>The site's artifact deployment state.</returns>
+    private static ArtifactDeploymentState ResolveArtifactState(
+        Site site,
+        IReadOnlyDictionary<int, string> currentArtifactHashes)
+    {
+        if (site.ArtifactRevisionHash is null)
+            return ArtifactDeploymentState.NeverDeployed;
+
+        return currentArtifactHashes.TryGetValue(site.Id, out var currentHash)
+               && string.Equals(site.ArtifactRevisionHash, currentHash, StringComparison.Ordinal)
+            ? ArtifactDeploymentState.Current
+            : ArtifactDeploymentState.OutOfDate;
+    }
+
+    /// <summary>
+    /// Counts one site's deployed, stale and un-evaluatable instances. Only instances
+    /// present in the staleness result are counted — the batch OMITS an instance with
+    /// no deployed snapshot, which is exactly the "nothing to be stale against"
+    /// exclusion the counts need.
+    /// </summary>
+    /// <param name="siteId">Site whose instances are being tallied.</param>
+    /// <param name="deployedInstances">All instances in a state other than NotDeployed.</param>
+    /// <param name="staleness">Instance id to stale (true) / current (false) / unknown (null).</param>
+    /// <returns>The deployed, stale and unknown counts for the site.</returns>
+    private static (int Deployed, int Stale, int Unknown) TallyInstances(
+        int siteId,
+        IReadOnlyList<Instance> deployedInstances,
+        IReadOnlyDictionary<int, bool?> staleness)
+    {
+        var deployed = 0;
+        var stale = 0;
+        var unknown = 0;
+
+        foreach (var instance in deployedInstances)
+        {
+            if (instance.SiteId != siteId)
+                continue;
+            if (!staleness.TryGetValue(instance.Id, out var isStale))
+                continue;
+
+            deployed++;
+            if (isStale == true)
+                stale++;
+            else if (isStale is null)
+                unknown++;
+        }
+
+        return (deployed, stale, unknown);
+    }
+}

src/ZB.MOM.WW.ScadaBridge.DeploymentManager/PendingUpdates/PendingUpdatesSnapshot.cs

@@ -0,0 +1,127 @@
+namespace ZB.MOM.WW.ScadaBridge.DeploymentManager.PendingUpdates;
+
+/// <summary>
+/// State of a site's system-wide artifact set (shared scripts, external systems,
+/// database connections, data connections) relative to the definitions central holds
+/// right now.
+/// </summary>
+public enum ArtifactDeploymentState
+{
+    /// <summary>
+    /// Artifacts have never been deployed to this site (the site row carries no
+    /// revision hash). Deliberately NOT a pending update: nothing has drifted, the
+    /// operator simply has not deployed yet. Surfaced on the Sites page so a
+    /// freshly-created site is obvious, never counted in the fleet totals or
+    /// escalated by the rail indicator.
+    /// </summary>
+    NeverDeployed,
+
+    /// <summary>
+    /// The hash stamped by the last successful artifact deploy equals the hash the
+    /// current definitions would produce — nothing to deploy.
+    /// </summary>
+    Current,
+
+    /// <summary>
+    /// The current definitions hash differently from what was last deployed, so the
+    /// site is running stale artifacts and a redeploy is pending.
+    /// </summary>
+    OutOfDate
+}
+
+/// <summary>
+/// One site's row in a <see cref="PendingUpdatesSnapshot"/>: its artifact state plus
+/// the instance staleness tallies that back the Sites-page badge and the rail
+/// indicator's breakdown.
+/// </summary>
+/// <param name="SiteId">Database identifier of the site.</param>
+/// <param name="SiteIdentifier">Machine-readable site identifier (as used in routing).</param>
+/// <param name="SiteName">Human-readable site display name.</param>
+/// <param name="ArtifactState">Whether this site's system-wide artifacts are current.</param>
+/// <param name="ArtifactsDeployedAt">
+/// UTC instant of the last successful artifact deploy to this site, or
+/// <see langword="null"/> when artifacts have never been deployed. Rendered as the
+/// badge tooltip.
+/// </param>
+/// <param name="DeployedInstanceCount">
+/// Instances at this site that HAVE a deployed configuration snapshot — the
+/// denominator of "N of M instances out of date". An instance the operator has never
+/// deployed has nothing to be stale against and is not counted here.
+/// </param>
+/// <param name="StaleInstanceCount">
+/// Deployed instances at this site whose current flatten differs from what is
+/// deployed, i.e. a redeploy is pending.
+/// </param>
+/// <param name="UnknownInstanceCount">
+/// Deployed instances at this site whose staleness could NOT be evaluated (their
+/// flatten failed — a missing template, a broken binding). Never counted as stale:
+/// the fleet totals must not invent a pending update out of a read failure.
+/// </param>
+public sealed record SitePendingUpdates(
+    int SiteId,
+    string SiteIdentifier,
+    string SiteName,
+    ArtifactDeploymentState ArtifactState,
+    DateTimeOffset? ArtifactsDeployedAt,
+    int DeployedInstanceCount,
+    int StaleInstanceCount,
+    int UnknownInstanceCount);
+
+/// <summary>
+/// One fleet-wide answer to "is anything waiting to be deployed?" — every site's
+/// artifact state and instance staleness tallies as of a single instant.
+///
+/// <para>
+/// Computed by <see cref="IPendingUpdatesEvaluator"/> and held behind the Central
+/// UI's single-flight TTL cache, so however many circuits are watching, the fleet is
+/// flattened at most once per TTL. An immutable value: a scoped view is taken with
+/// <see cref="ForSites"/> rather than by mutating the snapshot.
+/// </para>
+/// </summary>
+/// <param name="ComputedAt">
+/// UTC instant the snapshot was computed. Rendered as the "as of HH:mm UTC" caption
+/// and preserved verbatim through <see cref="ForSites"/> — a scoped view is the same
+/// observation, not a newer one.
+/// </param>
+/// <param name="Sites">Per-site rows, ordered by site name (ordinal, ignore-case).</param>
+public sealed record PendingUpdatesSnapshot(
+    DateTimeOffset ComputedAt,
+    IReadOnlyList<SitePendingUpdates> Sites)
+{
+    /// <summary>
+    /// Total instances across the snapshot's sites whose deployed configuration has
+    /// drifted from their current flatten. Instances whose staleness could not be
+    /// evaluated are excluded.
+    /// </summary>
+    public int StaleInstanceCount => Sites.Sum(s => s.StaleInstanceCount);
+
+    /// <summary>
+    /// Number of sites running out-of-date system-wide artifacts.
+    /// <see cref="ArtifactDeploymentState.NeverDeployed"/> is deliberately NOT counted
+    /// — the operator has not deployed yet, which is not a pending update.
+    /// </summary>
+    public int StaleArtifactSiteCount =>
+        Sites.Count(s => s.ArtifactState == ArtifactDeploymentState.OutOfDate);
+
+    /// <summary>
+    /// Whether anything in this snapshot is waiting to be deployed — the single
+    /// predicate the rail indicator switches on.
+    /// </summary>
+    public bool HasPendingUpdates => StaleInstanceCount > 0 || StaleArtifactSiteCount > 0;
+
+    /// <summary>
+    /// Returns a copy of this snapshot keeping only the named sites, preserving
+    /// <see cref="ComputedAt"/>. Used to narrow the fleet snapshot to a site-scoped
+    /// operator's permitted sites, so the computed totals never include a site the
+    /// user cannot see.
+    /// </summary>
+    /// <param name="siteIds">Database identifiers of the sites to keep.</param>
+    /// <returns>A snapshot over the retained sites, in the same order.</returns>
+    public PendingUpdatesSnapshot ForSites(IReadOnlySet<int> siteIds)
+    {
+        ArgumentNullException.ThrowIfNull(siteIds);
+        return new PendingUpdatesSnapshot(
+            ComputedAt,
+            Sites.Where(s => siteIds.Contains(s.SiteId)).ToList());
+    }
+}

src/ZB.MOM.WW.ScadaBridge.DeploymentManager/ServiceCollectionExtensions.cs

@@ -4,6 +4,7 @@ using Microsoft.Extensions.Options;
 using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
 using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Transport;
 using ZB.MOM.WW.ScadaBridge.Commons.Types.Templates;
+using ZB.MOM.WW.ScadaBridge.DeploymentManager.PendingUpdates;
 
 namespace ZB.MOM.WW.ScadaBridge.DeploymentManager;
 
@@ -69,6 +70,13 @@ public static class ServiceCollectionExtensions
         // flattened config has drifted from their deployed snapshot. Scoped so it
         // shares the per-request DbContext with the in-flight import.
         services.AddScoped<IStaleInstanceProbe, StaleInstanceProbe>();
+
+        // Fleet-wide "is anything waiting to be deployed?" aggregate behind the
+        // Central UI rail indicator and the Sites-page artifact badges. Scoped because
+        // it reads through the same per-request DbContext as the services it composes;
+        // the Central UI's singleton single-flight cache runs it in its own DI scope so
+        // the fleet is flattened at most once per cache TTL.
+        services.AddScoped<IPendingUpdatesEvaluator, PendingUpdatesEvaluator>();
         return services;
     }