feat(deploy): PendingUpdatesEvaluator — one fleet snapshot of stale instances and stale-artifact sites

One scoped read that answers "is anything waiting to be deployed?" for the whole
fleet: each site's stored artifact revision hash compared against what a deploy
would send right now (NeverDeployed / Current / OutOfDate), plus per-site
deployed/stale/unknown instance tallies from the batched staleness evaluation.
NeverDeployed is deliberately NOT a pending update — the operator has not deployed
yet — and exceptions propagate so the caller's single-flight cache can render
"unavailable" rather than invent an "up to date". Adds the validated
PendingUpdatesCacheTtl (60 s) that TTL will key off.

Claude-Session: https://claude.ai/code/session_01Y5ef34iUUMjmwnMpL9xgmB
This commit is contained in:
Joseph Doherty
2026-09-14 13:49:54 -04:00
parent 3b55d60ef9
commit fc48db480c
8 changed files with 789 additions and 0 deletions
@@ -75,4 +75,19 @@ public class DeploymentManagerOptions
/// </para> /// </para>
/// </summary> /// </summary>
public TimeSpan TerminalDeploymentRecordRetention { get; set; } = TimeSpan.FromDays(365); public TimeSpan TerminalDeploymentRecordRetention { get; set; } = TimeSpan.FromDays(365);
/// <summary>
/// How long the Central UI pending-updates snapshot stays fresh; one fleet
/// flatten per TTL while any Deployer/Administrator circuit is open.
///
/// <para>
/// The snapshot is a whole-fleet read (every site's artifact hash plus every
/// deployed instance's staleness), so it is held behind a single-flight TTL
/// cache: however many circuits poll, and however often, the fleet is evaluated
/// at most once per this window — and not at all when nobody is looking. Must be
/// positive; zero would make every poll a cache miss and re-flatten the fleet per
/// circuit, which is precisely the cost the cache exists to avoid.
/// </para>
/// </summary>
public TimeSpan PendingUpdatesCacheTtl { get; set; } = TimeSpan.FromSeconds(60);
} }
@@ -43,6 +43,12 @@ public sealed class DeploymentManagerOptionsValidator : OptionsValidatorBase<Dep
$"(was {options.SiteDeploymentReconcileTimeout}); it bounds each query-before-redeploy " + $"(was {options.SiteDeploymentReconcileTimeout}); it bounds each query-before-redeploy " +
$"reconciliation inside a bulk site deployment's serial prepare phase."); $"reconciliation inside a bulk site deployment's serial prepare phase.");
builder.RequireThat(options.PendingUpdatesCacheTtl > TimeSpan.Zero,
$"ScadaBridge:DeploymentManager:PendingUpdatesCacheTtl must be a positive duration " +
$"(was {options.PendingUpdatesCacheTtl}); it is the freshness window of the Central UI " +
$"pending-updates snapshot, and a non-positive value would re-flatten the whole fleet on " +
$"every poll of every open circuit.");
builder.RequireThat(options.TerminalDeploymentRecordRetention >= TimeSpan.Zero, builder.RequireThat(options.TerminalDeploymentRecordRetention >= TimeSpan.Zero,
$"ScadaBridge:DeploymentManager:TerminalDeploymentRecordRetention must not be negative " + $"ScadaBridge:DeploymentManager:TerminalDeploymentRecordRetention must not be negative " +
$"(was {options.TerminalDeploymentRecordRetention}); use TimeSpan.Zero to disable purging of " + $"(was {options.TerminalDeploymentRecordRetention}); use TimeSpan.Zero to disable purging of " +
@@ -0,0 +1,24 @@
namespace ZB.MOM.WW.ScadaBridge.DeploymentManager.PendingUpdates;
/// <summary>
/// Computes the fleet-wide "pending updates" aggregate — which sites are running
/// out-of-date system-wide artifacts and how many deployed instances have drifted
/// from their deployed configuration.
///
/// <para>
/// Scoped, and deliberately uncached at this seam: caching belongs to the Central
/// UI's single-flight TTL cache, which runs this evaluator in its own DI scope so
/// the fleet is flattened at most once per TTL regardless of how many circuits are
/// watching. Callers that resolve the evaluator directly pay a full fleet flatten
/// per call.
/// </para>
/// </summary>
public interface IPendingUpdatesEvaluator
{
/// <summary>
/// Evaluates every site and every deployed instance, returning one snapshot.
/// </summary>
/// <param name="cancellationToken">Cancellation token, honoured between instances.</param>
/// <returns>A snapshot of the fleet's pending updates as of now.</returns>
Task<PendingUpdatesSnapshot> EvaluateAsync(CancellationToken cancellationToken = default);
}
@@ -0,0 +1,172 @@
using System.Diagnostics;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.DeploymentManager.PendingUpdates;
/// <summary>
/// Default <see cref="IPendingUpdatesEvaluator"/>: one pass over the fleet that
/// answers both halves of "is anything waiting to be deployed?" — per-site artifact
/// staleness and per-instance configuration staleness.
///
/// <para>
/// <b>How it stays affordable.</b> Both halves are batch reads, not per-entity loops.
/// <see cref="ArtifactDeploymentService.ComputeCurrentArtifactHashesAsync"/> hoists
/// the global artifact queries once for the whole fleet, and
/// <see cref="DeploymentService.GetInstanceStalenessAsync"/> runs every flatten
/// through ONE <see cref="FlattenSession"/> so a template chain shared by N
/// instances is walked once. It is still a whole-fleet flatten, which is why the
/// caller caches it behind a TTL rather than calling it per request.
/// </para>
///
/// <para>
/// <b>Exceptions propagate.</b> Nothing is swallowed and no partial snapshot is
/// invented: the caller's single-flight cache discards a faulted flight and the UI
/// renders <i>unavailable</i>. Degrading a failed read to "up to date" is the one
/// answer that must never be manufactured — it would silently hide a fleet waiting
/// on a deploy.
/// </para>
/// </summary>
public sealed class PendingUpdatesEvaluator : IPendingUpdatesEvaluator
{
private readonly ISiteRepository _siteRepository;
private readonly ITemplateEngineRepository _templateRepository;
private readonly ArtifactDeploymentService _artifactDeploymentService;
private readonly DeploymentService _deploymentService;
private readonly ILogger<PendingUpdatesEvaluator> _logger;
/// <summary>
/// Initializes a new evaluator.
/// </summary>
/// <param name="siteRepository">Repository the site list is read from.</param>
/// <param name="templateRepository">Repository the instance list is read from.</param>
/// <param name="artifactDeploymentService">Computes each site's current artifact revision hash.</param>
/// <param name="deploymentService">Evaluates instance staleness in one batched flatten session.</param>
/// <param name="logger">Logger instance.</param>
public PendingUpdatesEvaluator(
ISiteRepository siteRepository,
ITemplateEngineRepository templateRepository,
ArtifactDeploymentService artifactDeploymentService,
DeploymentService deploymentService,
ILogger<PendingUpdatesEvaluator> logger)
{
_siteRepository = siteRepository;
_templateRepository = templateRepository;
_artifactDeploymentService = artifactDeploymentService;
_deploymentService = deploymentService;
_logger = logger;
}
/// <inheritdoc />
public async Task<PendingUpdatesSnapshot> EvaluateAsync(CancellationToken cancellationToken = default)
{
var stopwatch = Stopwatch.StartNew();
var sites = await _siteRepository.GetAllSitesAsync(cancellationToken);
var instances = await _templateRepository.GetAllInstancesAsync(cancellationToken);
var currentArtifactHashes =
await _artifactDeploymentService.ComputeCurrentArtifactHashesAsync(sites, cancellationToken);
// Only instances the operator has actually deployed can be stale; a
// NotDeployed instance has nothing to compare against, and flattening it
// would be pure cost on every TTL expiry.
var deployedInstances = instances.Where(i => i.State != InstanceState.NotDeployed).ToList();
var staleness = await _deploymentService.GetInstanceStalenessAsync(
deployedInstances.Select(i => i.Id).ToList(), cancellationToken);
var rows = new List<SitePendingUpdates>(sites.Count);
foreach (var site in sites.OrderBy(s => s.Name, StringComparer.OrdinalIgnoreCase))
{
var (deployed, stale, unknown) = TallyInstances(site.Id, deployedInstances, staleness);
rows.Add(new SitePendingUpdates(
site.Id,
site.SiteIdentifier,
site.Name,
ResolveArtifactState(site, currentArtifactHashes),
site.ArtifactsDeployedAt,
deployed,
stale,
unknown));
}
var snapshot = new PendingUpdatesSnapshot(DateTimeOffset.UtcNow, rows);
// One line per evaluation — this runs at most once per cache TTL, so it is
// the cheapest record of what the fleet looked like and what the sweep cost.
_logger.LogInformation(
"Pending-updates snapshot: {SiteCount} site(s), {DeployedInstanceCount} deployed instance(s), " +
"{StaleInstanceCount} stale instance(s), {StaleArtifactSiteCount} site(s) with out-of-date " +
"artifacts, in {ElapsedMs} ms",
rows.Count,
rows.Sum(r => r.DeployedInstanceCount),
snapshot.StaleInstanceCount,
snapshot.StaleArtifactSiteCount,
stopwatch.ElapsedMilliseconds);
return snapshot;
}
/// <summary>
/// Decides a site's artifact state by comparing the hash stamped by its last
/// successful artifact deploy against the hash the current definitions would
/// produce. A site with no stamp has never had artifacts deployed, which is
/// <see cref="ArtifactDeploymentState.NeverDeployed"/>, NOT out of date. A site
/// missing from the computed map (it appeared between the two reads) is reported
/// out of date, the side that prompts an operator to look rather than hiding it.
/// </summary>
/// <param name="site">The site to classify.</param>
/// <param name="currentArtifactHashes">Site id to the hash a deploy would send right now.</param>
/// <returns>The site's artifact deployment state.</returns>
private static ArtifactDeploymentState ResolveArtifactState(
Site site,
IReadOnlyDictionary<int, string> currentArtifactHashes)
{
if (site.ArtifactRevisionHash is null)
return ArtifactDeploymentState.NeverDeployed;
return currentArtifactHashes.TryGetValue(site.Id, out var currentHash)
&& string.Equals(site.ArtifactRevisionHash, currentHash, StringComparison.Ordinal)
? ArtifactDeploymentState.Current
: ArtifactDeploymentState.OutOfDate;
}
/// <summary>
/// Counts one site's deployed, stale and un-evaluatable instances. Only instances
/// present in the staleness result are counted — the batch OMITS an instance with
/// no deployed snapshot, which is exactly the "nothing to be stale against"
/// exclusion the counts need.
/// </summary>
/// <param name="siteId">Site whose instances are being tallied.</param>
/// <param name="deployedInstances">All instances in a state other than NotDeployed.</param>
/// <param name="staleness">Instance id to stale (true) / current (false) / unknown (null).</param>
/// <returns>The deployed, stale and unknown counts for the site.</returns>
private static (int Deployed, int Stale, int Unknown) TallyInstances(
int siteId,
IReadOnlyList<Instance> deployedInstances,
IReadOnlyDictionary<int, bool?> staleness)
{
var deployed = 0;
var stale = 0;
var unknown = 0;
foreach (var instance in deployedInstances)
{
if (instance.SiteId != siteId)
continue;
if (!staleness.TryGetValue(instance.Id, out var isStale))
continue;
deployed++;
if (isStale == true)
stale++;
else if (isStale is null)
unknown++;
}
return (deployed, stale, unknown);
}
}
@@ -0,0 +1,127 @@
namespace ZB.MOM.WW.ScadaBridge.DeploymentManager.PendingUpdates;
/// <summary>
/// State of a site's system-wide artifact set (shared scripts, external systems,
/// database connections, data connections) relative to the definitions central holds
/// right now.
/// </summary>
public enum ArtifactDeploymentState
{
/// <summary>
/// Artifacts have never been deployed to this site (the site row carries no
/// revision hash). Deliberately NOT a pending update: nothing has drifted, the
/// operator simply has not deployed yet. Surfaced on the Sites page so a
/// freshly-created site is obvious, never counted in the fleet totals or
/// escalated by the rail indicator.
/// </summary>
NeverDeployed,
/// <summary>
/// The hash stamped by the last successful artifact deploy equals the hash the
/// current definitions would produce — nothing to deploy.
/// </summary>
Current,
/// <summary>
/// The current definitions hash differently from what was last deployed, so the
/// site is running stale artifacts and a redeploy is pending.
/// </summary>
OutOfDate
}
/// <summary>
/// One site's row in a <see cref="PendingUpdatesSnapshot"/>: its artifact state plus
/// the instance staleness tallies that back the Sites-page badge and the rail
/// indicator's breakdown.
/// </summary>
/// <param name="SiteId">Database identifier of the site.</param>
/// <param name="SiteIdentifier">Machine-readable site identifier (as used in routing).</param>
/// <param name="SiteName">Human-readable site display name.</param>
/// <param name="ArtifactState">Whether this site's system-wide artifacts are current.</param>
/// <param name="ArtifactsDeployedAt">
/// UTC instant of the last successful artifact deploy to this site, or
/// <see langword="null"/> when artifacts have never been deployed. Rendered as the
/// badge tooltip.
/// </param>
/// <param name="DeployedInstanceCount">
/// Instances at this site that HAVE a deployed configuration snapshot — the
/// denominator of "N of M instances out of date". An instance the operator has never
/// deployed has nothing to be stale against and is not counted here.
/// </param>
/// <param name="StaleInstanceCount">
/// Deployed instances at this site whose current flatten differs from what is
/// deployed, i.e. a redeploy is pending.
/// </param>
/// <param name="UnknownInstanceCount">
/// Deployed instances at this site whose staleness could NOT be evaluated (their
/// flatten failed — a missing template, a broken binding). Never counted as stale:
/// the fleet totals must not invent a pending update out of a read failure.
/// </param>
public sealed record SitePendingUpdates(
int SiteId,
string SiteIdentifier,
string SiteName,
ArtifactDeploymentState ArtifactState,
DateTimeOffset? ArtifactsDeployedAt,
int DeployedInstanceCount,
int StaleInstanceCount,
int UnknownInstanceCount);
/// <summary>
/// One fleet-wide answer to "is anything waiting to be deployed?" — every site's
/// artifact state and instance staleness tallies as of a single instant.
///
/// <para>
/// Computed by <see cref="IPendingUpdatesEvaluator"/> and held behind the Central
/// UI's single-flight TTL cache, so however many circuits are watching, the fleet is
/// flattened at most once per TTL. An immutable value: a scoped view is taken with
/// <see cref="ForSites"/> rather than by mutating the snapshot.
/// </para>
/// </summary>
/// <param name="ComputedAt">
/// UTC instant the snapshot was computed. Rendered as the "as of HH:mm UTC" caption
/// and preserved verbatim through <see cref="ForSites"/> — a scoped view is the same
/// observation, not a newer one.
/// </param>
/// <param name="Sites">Per-site rows, ordered by site name (ordinal, ignore-case).</param>
public sealed record PendingUpdatesSnapshot(
DateTimeOffset ComputedAt,
IReadOnlyList<SitePendingUpdates> Sites)
{
/// <summary>
/// Total instances across the snapshot's sites whose deployed configuration has
/// drifted from their current flatten. Instances whose staleness could not be
/// evaluated are excluded.
/// </summary>
public int StaleInstanceCount => Sites.Sum(s => s.StaleInstanceCount);
/// <summary>
/// Number of sites running out-of-date system-wide artifacts.
/// <see cref="ArtifactDeploymentState.NeverDeployed"/> is deliberately NOT counted
/// — the operator has not deployed yet, which is not a pending update.
/// </summary>
public int StaleArtifactSiteCount =>
Sites.Count(s => s.ArtifactState == ArtifactDeploymentState.OutOfDate);
/// <summary>
/// Whether anything in this snapshot is waiting to be deployed — the single
/// predicate the rail indicator switches on.
/// </summary>
public bool HasPendingUpdates => StaleInstanceCount > 0 || StaleArtifactSiteCount > 0;
/// <summary>
/// Returns a copy of this snapshot keeping only the named sites, preserving
/// <see cref="ComputedAt"/>. Used to narrow the fleet snapshot to a site-scoped
/// operator's permitted sites, so the computed totals never include a site the
/// user cannot see.
/// </summary>
/// <param name="siteIds">Database identifiers of the sites to keep.</param>
/// <returns>A snapshot over the retained sites, in the same order.</returns>
public PendingUpdatesSnapshot ForSites(IReadOnlySet<int> siteIds)
{
ArgumentNullException.ThrowIfNull(siteIds);
return new PendingUpdatesSnapshot(
ComputedAt,
Sites.Where(s => siteIds.Contains(s.SiteId)).ToList());
}
}
@@ -4,6 +4,7 @@ using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Transport; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Transport;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Templates; using ZB.MOM.WW.ScadaBridge.Commons.Types.Templates;
using ZB.MOM.WW.ScadaBridge.DeploymentManager.PendingUpdates;
namespace ZB.MOM.WW.ScadaBridge.DeploymentManager; namespace ZB.MOM.WW.ScadaBridge.DeploymentManager;
@@ -69,6 +70,13 @@ public static class ServiceCollectionExtensions
// flattened config has drifted from their deployed snapshot. Scoped so it // flattened config has drifted from their deployed snapshot. Scoped so it
// shares the per-request DbContext with the in-flight import. // shares the per-request DbContext with the in-flight import.
services.AddScoped<IStaleInstanceProbe, StaleInstanceProbe>(); services.AddScoped<IStaleInstanceProbe, StaleInstanceProbe>();
// Fleet-wide "is anything waiting to be deployed?" aggregate behind the
// Central UI rail indicator and the Sites-page artifact badges. Scoped because
// it reads through the same per-request DbContext as the services it composes;
// the Central UI's singleton single-flight cache runs it in its own DI scope so
// the fleet is flattened at most once per cache TTL.
services.AddScoped<IPendingUpdatesEvaluator, PendingUpdatesEvaluator>();
return services; return services;
} }
@@ -72,4 +72,26 @@ public class DeploymentManagerOptionsValidatorTests
Assert.True(result.Failed); Assert.True(result.Failed);
Assert.Contains("SiteDeploymentReconcileTimeout", result.FailureMessage); Assert.Contains("SiteDeploymentReconcileTimeout", result.FailureMessage);
} }
[Fact]
public void ZeroPendingUpdatesCacheTtl_IsRejected()
{
// A zero TTL makes every Deployer/Administrator circuit's poll a cache miss,
// so the whole fleet is re-flattened on each one — the exact cost the cache
// exists to avoid.
var result = Validate(new DeploymentManagerOptions { PendingUpdatesCacheTtl = TimeSpan.Zero });
Assert.True(result.Failed);
Assert.Contains("PendingUpdatesCacheTtl", result.FailureMessage);
}
[Fact]
public void NegativePendingUpdatesCacheTtl_IsRejected()
{
var result = Validate(new DeploymentManagerOptions
{
PendingUpdatesCacheTtl = TimeSpan.FromSeconds(-1)
});
Assert.True(result.Failed);
Assert.Contains("PendingUpdatesCacheTtl", result.FailureMessage);
}
} }
@@ -0,0 +1,415 @@
using System.Text.Json;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Transport;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Templates;
using ZB.MOM.WW.ScadaBridge.Communication;
using ZB.MOM.WW.ScadaBridge.DeploymentManager.PendingUpdates;
using ZB.MOM.WW.ScadaBridge.TemplateEngine.Flattening;
using ZB.MOM.WW.ScadaBridge.TemplateEngine.Validation;
namespace ZB.MOM.WW.ScadaBridge.DeploymentManager.Tests;
/// <summary>
/// Coverage for <see cref="PendingUpdatesEvaluator"/> — the single fleet-wide read
/// behind the Central UI rail indicator and the Sites page badges.
///
/// <para>
/// The evaluator is built over the REAL <see cref="ArtifactDeploymentService"/> and
/// <see cref="DeploymentService"/> (substituting only their repository and flattening
/// seams, as the sibling suites do), because the two properties worth pinning are
/// exactly the ones a mocked collaborator would paper over: that the artifact state
/// is decided by comparing the site's STORED hash against a hash recomputed from the
/// current definitions, and that only instances the operator has actually deployed
/// are ever handed to the staleness batch.
/// </para>
/// </summary>
public class PendingUpdatesEvaluatorTests
{
private readonly ISiteRepository _siteRepo = Substitute.For<ISiteRepository>();
private readonly ITemplateEngineRepository _templateRepo = Substitute.For<ITemplateEngineRepository>();
private readonly IExternalSystemRepository _externalSystemRepo = Substitute.For<IExternalSystemRepository>();
private readonly INotificationRepository _notificationRepo = Substitute.For<INotificationRepository>();
private readonly IDeploymentManagerRepository _deploymentRepo = Substitute.For<IDeploymentManagerRepository>();
private readonly IFlatteningPipeline _pipeline = Substitute.For<IFlatteningPipeline>();
private readonly IAuditService _audit = Substitute.For<IAuditService>();
private readonly RevisionHashService _hasher = new();
private readonly ArtifactRevisionHashService _artifactHasher = new();
private readonly ArtifactDeploymentService _artifactService;
private readonly DeploymentService _deploymentService;
private readonly PendingUpdatesEvaluator _evaluator;
/// <summary>
/// Builds the evaluator over the real services; every repository and the
/// flattening pipeline are substituted.
/// </summary>
public PendingUpdatesEvaluatorTests()
{
_pipeline.CreateSession().Returns(new FlattenSession(new TemplateGraphWatermark()));
var comms = new CommunicationService(
Options.Create(new CommunicationOptions()),
NullLogger<CommunicationService>.Instance);
_artifactService = new ArtifactDeploymentService(
_siteRepo, _deploymentRepo, _templateRepo, _externalSystemRepo, _notificationRepo,
comms, _audit,
Options.Create(new DeploymentManagerOptions()),
NullLogger<ArtifactDeploymentService>.Instance,
_artifactHasher);
_deploymentService = new DeploymentService(
_deploymentRepo,
_siteRepo,
_pipeline,
comms,
new OperationLockManager(),
_audit,
new DiffService(),
_hasher,
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
Substitute.For<IStaleInstanceProbe>(),
Options.Create(new DeploymentManagerOptions()),
Options.Create(new CommunicationOptions()),
NullLogger<DeploymentService>.Instance);
_evaluator = new PendingUpdatesEvaluator(
_siteRepo, _templateRepo, _artifactService, _deploymentService,
NullLogger<PendingUpdatesEvaluator>.Instance);
}
// ── arrange helpers ──────────────────────────────────────────────────────
/// <summary>Stubs the site list read.</summary>
private void ArrangeSites(params Site[] sites) =>
_siteRepo.GetAllSitesAsync(Arg.Any<CancellationToken>()).Returns(sites);
/// <summary>Stubs the instance list read.</summary>
private void ArrangeInstances(params Instance[] instances) =>
_templateRepo.GetAllInstancesAsync(Arg.Any<CancellationToken>()).Returns(instances);
/// <summary>Builds an instance in the given lifecycle state at the given site.</summary>
private static Instance MakeInstance(int id, int siteId, InstanceState state) =>
new($"Inst{id}") { Id = id, SiteId = siteId, State = state };
/// <summary>Builds a one-attribute flattened config with the given scalar value.</summary>
private static FlattenedConfiguration ScalarConfig(string value) => new()
{
InstanceUniqueName = "TestInst",
Attributes = [new ResolvedAttribute { CanonicalName = "Setpoint", Value = value, DataType = "Int32" }]
};
/// <summary>Stubs a successful flatten of <paramref name="instanceId"/> at the given hash.</summary>
private void ArrangeFlatten(int instanceId, FlattenedConfiguration config, string hash) =>
_pipeline.FlattenAndValidateAsync(
instanceId, Arg.Any<CancellationToken>(), Arg.Any<bool>(), Arg.Any<FlattenSession?>())
.Returns(Result<FlatteningPipelineResult>.Success(
new FlatteningPipelineResult(config, hash, ValidationResult.Success())));
/// <summary>Stubs a FAILED flatten of <paramref name="instanceId"/> — the "unknown" leg.</summary>
private void ArrangeFlattenFailure(int instanceId) =>
_pipeline.FlattenAndValidateAsync(
instanceId, Arg.Any<CancellationToken>(), Arg.Any<bool>(), Arg.Any<FlattenSession?>())
.Returns(Result<FlatteningPipelineResult>.Failure("template missing"));
/// <summary>Stubs the (InstanceId, RevisionHash) projection read.</summary>
private void ArrangeDeployedHashes(params (int InstanceId, string Hash)[] rows) =>
_deploymentRepo.GetDeployedRevisionHashesAsync(Arg.Any<CancellationToken>())
.Returns(rows.ToDictionary(r => r.InstanceId, r => r.Hash));
/// <summary>Stubs the fat snapshot read for <paramref name="instanceId"/>.</summary>
private void ArrangeSnapshot(int instanceId, FlattenedConfiguration config, string frozenHash) =>
_deploymentRepo.GetDeployedSnapshotByInstanceIdAsync(instanceId, Arg.Any<CancellationToken>())
.Returns(new DeployedConfigSnapshot("dep1", frozenHash, JsonSerializer.Serialize(config))
{
InstanceId = instanceId,
DeployedAt = DateTimeOffset.UtcNow
});
/// <summary>
/// Asks the artifact service what the site's hash WOULD be right now — the value a
/// successful deploy would have stamped, i.e. what makes a site read Current.
/// </summary>
private async Task<string> CurrentArtifactHashAsync(Site site) =>
(await _artifactService.ComputeCurrentArtifactHashesAsync([site]))[site.Id];
// ── artifact state ───────────────────────────────────────────────────────
[Fact]
public async Task EvaluateAsync_NullStoredHash_IsNeverDeployedAndNotAPendingUpdate()
{
var site = new Site("Alpha", "site-a") { Id = 1 };
ArrangeSites(site);
ArrangeInstances();
var snapshot = await _evaluator.EvaluateAsync();
var row = Assert.Single(snapshot.Sites);
Assert.Equal(ArtifactDeploymentState.NeverDeployed, row.ArtifactState);
Assert.Null(row.ArtifactsDeployedAt);
// NeverDeployed is the operator not having deployed yet, not a pending
// update: it is surfaced on the Sites page but never counted or nagged about.
Assert.Equal(0, snapshot.StaleArtifactSiteCount);
Assert.False(snapshot.HasPendingUpdates);
}
[Fact]
public async Task EvaluateAsync_StoredHashMatchesCurrentDefinitions_IsCurrent()
{
var site = new Site("Alpha", "site-a") { Id = 1 };
var deployedAt = DateTimeOffset.UtcNow.AddHours(-2);
site.ArtifactRevisionHash = await CurrentArtifactHashAsync(site);
site.ArtifactsDeployedAt = deployedAt;
ArrangeSites(site);
ArrangeInstances();
var snapshot = await _evaluator.EvaluateAsync();
var row = Assert.Single(snapshot.Sites);
Assert.Equal(ArtifactDeploymentState.Current, row.ArtifactState);
Assert.Equal(deployedAt, row.ArtifactsDeployedAt);
Assert.Equal(0, snapshot.StaleArtifactSiteCount);
Assert.False(snapshot.HasPendingUpdates);
}
[Fact]
public async Task EvaluateAsync_StoredHashDiffersFromCurrentDefinitions_IsOutOfDateAndCounted()
{
var site = new Site("Alpha", "site-a") { Id = 1 };
site.ArtifactRevisionHash = "a-hash-from-an-older-artifact-set";
site.ArtifactsDeployedAt = DateTimeOffset.UtcNow.AddDays(-3);
ArrangeSites(site);
ArrangeInstances();
var snapshot = await _evaluator.EvaluateAsync();
var row = Assert.Single(snapshot.Sites);
Assert.Equal(ArtifactDeploymentState.OutOfDate, row.ArtifactState);
Assert.Equal(1, snapshot.StaleArtifactSiteCount);
Assert.True(snapshot.HasPendingUpdates);
}
[Fact]
public async Task EvaluateAsync_CarriesSiteIdentityOntoTheRow()
{
var site = new Site("Alpha Plant", "site-a") { Id = 7 };
ArrangeSites(site);
ArrangeInstances();
var snapshot = await _evaluator.EvaluateAsync();
var row = Assert.Single(snapshot.Sites);
Assert.Equal(7, row.SiteId);
Assert.Equal("site-a", row.SiteIdentifier);
Assert.Equal("Alpha Plant", row.SiteName);
}
// ── instance counts ──────────────────────────────────────────────────────
[Fact]
public async Task EvaluateAsync_CountsDeployedStaleAndUnknownInstancesPerSite()
{
var site = new Site("Alpha", "site-a") { Id = 1 };
ArrangeSites(site);
// 1 = deployed + stale, 2 = deployed but un-evaluatable, 3 = never deployed.
ArrangeInstances(
MakeInstance(1, 1, InstanceState.Enabled),
MakeInstance(2, 1, InstanceState.Disabled),
MakeInstance(3, 1, InstanceState.NotDeployed));
var deployedConfig = ScalarConfig("5");
var deployedHash = _hasher.ComputeHash(deployedConfig);
ArrangeDeployedHashes((1, deployedHash), (2, deployedHash));
ArrangeFlatten(1, ScalarConfig("9"), _hasher.ComputeHash(ScalarConfig("9")));
ArrangeSnapshot(1, deployedConfig, deployedHash);
ArrangeFlattenFailure(2);
var snapshot = await _evaluator.EvaluateAsync();
var row = Assert.Single(snapshot.Sites);
Assert.Equal(2, row.DeployedInstanceCount);
Assert.Equal(1, row.StaleInstanceCount);
Assert.Equal(1, row.UnknownInstanceCount);
Assert.Equal(1, snapshot.StaleInstanceCount);
Assert.True(snapshot.HasPendingUpdates);
}
[Fact]
public async Task EvaluateAsync_NotDeployedInstancesAreNeverHandedToTheStalenessBatch()
{
var site = new Site("Alpha", "site-a") { Id = 1 };
ArrangeSites(site);
ArrangeInstances(
MakeInstance(1, 1, InstanceState.Enabled),
MakeInstance(2, 1, InstanceState.NotDeployed));
var config = ScalarConfig("5");
var hash = _hasher.ComputeHash(config);
// Instance 2 deliberately HAS a leftover deployed-hash row (an undeploy that
// left the snapshot behind), so the staleness batch would happily evaluate it
// if its id were passed — the filter under test is the State gate, not the
// batch's own omission of instances with no snapshot.
ArrangeDeployedHashes((1, hash), (2, hash));
ArrangeFlatten(1, config, hash);
ArrangeFlatten(2, config, hash);
var snapshot = await _evaluator.EvaluateAsync();
// A NotDeployed instance has nothing to be stale against; flattening it would
// be pure cost on every TTL expiry, so it must never reach the batch.
await _pipeline.Received(1).FlattenAndValidateAsync(
1, Arg.Any<CancellationToken>(), Arg.Any<bool>(), Arg.Any<FlattenSession?>());
await _pipeline.DidNotReceive().FlattenAndValidateAsync(
2, Arg.Any<CancellationToken>(), Arg.Any<bool>(), Arg.Any<FlattenSession?>());
Assert.Equal(1, snapshot.Sites.Single().DeployedInstanceCount);
}
[Fact]
public async Task EvaluateAsync_AttributesInstanceCountsToTheirOwnSite()
{
var alpha = new Site("Alpha", "site-a") { Id = 1 };
var bravo = new Site("Bravo", "site-b") { Id = 2 };
ArrangeSites(alpha, bravo);
ArrangeInstances(
MakeInstance(1, 1, InstanceState.Enabled),
MakeInstance(2, 2, InstanceState.Enabled));
var deployedConfig = ScalarConfig("5");
var deployedHash = _hasher.ComputeHash(deployedConfig);
ArrangeDeployedHashes((1, deployedHash), (2, deployedHash));
ArrangeFlatten(1, deployedConfig, deployedHash);
ArrangeFlatten(2, ScalarConfig("9"), _hasher.ComputeHash(ScalarConfig("9")));
ArrangeSnapshot(2, deployedConfig, deployedHash);
var snapshot = await _evaluator.EvaluateAsync();
Assert.Equal(0, snapshot.Sites.Single(s => s.SiteId == 1).StaleInstanceCount);
Assert.Equal(1, snapshot.Sites.Single(s => s.SiteId == 2).StaleInstanceCount);
Assert.Equal(1, snapshot.StaleInstanceCount);
}
[Fact]
public async Task EvaluateAsync_NothingStale_HasNoPendingUpdates()
{
var site = new Site("Alpha", "site-a") { Id = 1 };
site.ArtifactRevisionHash = await CurrentArtifactHashAsync(site);
ArrangeSites(site);
ArrangeInstances(MakeInstance(1, 1, InstanceState.Enabled));
var config = ScalarConfig("5");
var hash = _hasher.ComputeHash(config);
ArrangeDeployedHashes((1, hash));
ArrangeFlatten(1, config, hash);
var snapshot = await _evaluator.EvaluateAsync();
Assert.Equal(1, snapshot.Sites.Single().DeployedInstanceCount);
Assert.Equal(0, snapshot.StaleInstanceCount);
Assert.Equal(0, snapshot.StaleArtifactSiteCount);
Assert.False(snapshot.HasPendingUpdates);
}
// ── ordering, ComputedAt, ForSites ───────────────────────────────────────
[Fact]
public async Task EvaluateAsync_OrdersSitesByNameIgnoringCase()
{
ArrangeSites(
new Site("charlie", "site-c") { Id = 3 },
new Site("Alpha", "site-a") { Id = 1 },
new Site("bravo", "site-b") { Id = 2 });
ArrangeInstances();
var snapshot = await _evaluator.EvaluateAsync();
Assert.Equal(["Alpha", "bravo", "charlie"], snapshot.Sites.Select(s => s.SiteName));
}
[Fact]
public async Task EvaluateAsync_StampsComputedAt()
{
var before = DateTimeOffset.UtcNow;
ArrangeSites(new Site("Alpha", "site-a") { Id = 1 });
ArrangeInstances();
var snapshot = await _evaluator.EvaluateAsync();
Assert.InRange(snapshot.ComputedAt, before, DateTimeOffset.UtcNow);
}
[Fact]
public async Task EvaluateAsync_ExceptionsPropagate()
{
_siteRepo.GetAllSitesAsync(Arg.Any<CancellationToken>())
.Returns<IReadOnlyList<Site>>(_ => throw new InvalidOperationException("db down"));
// The caller's single-flight cache discards a faulted flight and the UI
// renders "unavailable"; swallowing here would make a broken fleet read
// "up to date", which is the one answer that must never be invented.
var ex = await Assert.ThrowsAsync<InvalidOperationException>(() => _evaluator.EvaluateAsync());
Assert.Equal("db down", ex.Message);
}
[Fact]
public void ForSites_KeepsOnlyTheNamedSitesAndTheSameComputedAt()
{
var computedAt = DateTimeOffset.UtcNow.AddMinutes(-1);
var snapshot = new PendingUpdatesSnapshot(computedAt,
[
new SitePendingUpdates(1, "site-a", "Alpha", ArtifactDeploymentState.OutOfDate, null, 2, 2, 0),
new SitePendingUpdates(2, "site-b", "Bravo", ArtifactDeploymentState.Current, null, 1, 1, 0),
new SitePendingUpdates(3, "site-c", "Charlie", ArtifactDeploymentState.OutOfDate, null, 0, 0, 0)
]);
var scoped = snapshot.ForSites(new HashSet<int> { 2 });
Assert.Equal(computedAt, scoped.ComputedAt);
var row = Assert.Single(scoped.Sites);
Assert.Equal(2, row.SiteId);
// The totals are recomputed over the kept sites, so a site-scoped user is
// never shown a count that includes a site they cannot see.
Assert.Equal(1, scoped.StaleInstanceCount);
Assert.Equal(0, scoped.StaleArtifactSiteCount);
Assert.True(scoped.HasPendingUpdates);
}
[Fact]
public void ForSites_EmptyScope_YieldsNoPendingUpdates()
{
var snapshot = new PendingUpdatesSnapshot(DateTimeOffset.UtcNow,
[
new SitePendingUpdates(1, "site-a", "Alpha", ArtifactDeploymentState.OutOfDate, null, 2, 2, 0)
]);
var scoped = snapshot.ForSites(new HashSet<int>());
Assert.Empty(scoped.Sites);
Assert.False(scoped.HasPendingUpdates);
}
[Fact]
public void Snapshot_Totals_SumAcrossSites()
{
var snapshot = new PendingUpdatesSnapshot(DateTimeOffset.UtcNow,
[
new SitePendingUpdates(1, "site-a", "Alpha", ArtifactDeploymentState.OutOfDate, null, 4, 2, 1),
new SitePendingUpdates(2, "site-b", "Bravo", ArtifactDeploymentState.NeverDeployed, null, 0, 0, 0),
new SitePendingUpdates(3, "site-c", "Charlie", ArtifactDeploymentState.Current, null, 3, 3, 0)
]);
Assert.Equal(5, snapshot.StaleInstanceCount);
// NeverDeployed is deliberately not counted alongside OutOfDate.
Assert.Equal(1, snapshot.StaleArtifactSiteCount);
Assert.True(snapshot.HasPendingUpdates);
}
}