feat(deploy): PendingUpdatesEvaluator — one fleet snapshot of stale instances and stale-artifact sites
One scoped read that answers "is anything waiting to be deployed?" for the whole fleet: each site's stored artifact revision hash compared against what a deploy would send right now (NeverDeployed / Current / OutOfDate), plus per-site deployed/stale/unknown instance tallies from the batched staleness evaluation. NeverDeployed is deliberately NOT a pending update — the operator has not deployed yet — and exceptions propagate so the caller's single-flight cache can render "unavailable" rather than invent an "up to date". Adds the validated PendingUpdatesCacheTtl (60 s) that TTL will key off. Claude-Session: https://claude.ai/code/session_01Y5ef34iUUMjmwnMpL9xgmB
This commit is contained in:
Joseph Doherty
@@ -75,4 +75,19 @@ public class DeploymentManagerOptions
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public TimeSpan TerminalDeploymentRecordRetention { get; set; } = TimeSpan.FromDays(365);
|
||||
|
||||
/// <summary>
|
||||
/// How long the Central UI pending-updates snapshot stays fresh; one fleet
|
||||
/// flatten per TTL while any Deployer/Administrator circuit is open.
|
||||
///
|
||||
/// <para>
|
||||
/// The snapshot is a whole-fleet read (every site's artifact hash plus every
|
||||
/// deployed instance's staleness), so it is held behind a single-flight TTL
|
||||
/// cache: however many circuits poll, and however often, the fleet is evaluated
|
||||
/// at most once per this window — and not at all when nobody is looking. Must be
|
||||
/// positive; zero would make every poll a cache miss and re-flatten the fleet per
|
||||
/// circuit, which is precisely the cost the cache exists to avoid.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public TimeSpan PendingUpdatesCacheTtl { get; set; } = TimeSpan.FromSeconds(60);
|
||||
}
|
||||
|
||||
@@ -43,6 +43,12 @@ public sealed class DeploymentManagerOptionsValidator : OptionsValidatorBase<Dep
|
||||
$"(was {options.SiteDeploymentReconcileTimeout}); it bounds each query-before-redeploy " +
|
||||
$"reconciliation inside a bulk site deployment's serial prepare phase.");
|
||||
|
||||
builder.RequireThat(options.PendingUpdatesCacheTtl > TimeSpan.Zero,
|
||||
$"ScadaBridge:DeploymentManager:PendingUpdatesCacheTtl must be a positive duration " +
|
||||
$"(was {options.PendingUpdatesCacheTtl}); it is the freshness window of the Central UI " +
|
||||
$"pending-updates snapshot, and a non-positive value would re-flatten the whole fleet on " +
|
||||
$"every poll of every open circuit.");
|
||||
|
||||
builder.RequireThat(options.TerminalDeploymentRecordRetention >= TimeSpan.Zero,
|
||||
$"ScadaBridge:DeploymentManager:TerminalDeploymentRecordRetention must not be negative " +
|
||||
$"(was {options.TerminalDeploymentRecordRetention}); use TimeSpan.Zero to disable purging of " +
|
||||
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.DeploymentManager.PendingUpdates;
|
||||
|
||||
/// <summary>
|
||||
/// Computes the fleet-wide "pending updates" aggregate — which sites are running
|
||||
/// out-of-date system-wide artifacts and how many deployed instances have drifted
|
||||
/// from their deployed configuration.
|
||||
///
|
||||
/// <para>
|
||||
/// Scoped, and deliberately uncached at this seam: caching belongs to the Central
|
||||
/// UI's single-flight TTL cache, which runs this evaluator in its own DI scope so
|
||||
/// the fleet is flattened at most once per TTL regardless of how many circuits are
|
||||
/// watching. Callers that resolve the evaluator directly pay a full fleet flatten
|
||||
/// per call.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public interface IPendingUpdatesEvaluator
|
||||
{
|
||||
/// <summary>
|
||||
/// Evaluates every site and every deployed instance, returning one snapshot.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Cancellation token, honoured between instances.</param>
|
||||
/// <returns>A snapshot of the fleet's pending updates as of now.</returns>
|
||||
Task<PendingUpdatesSnapshot> EvaluateAsync(CancellationToken cancellationToken = default);
|
||||
}
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.DeploymentManager.PendingUpdates;
|
||||
|
||||
/// <summary>
|
||||
/// Default <see cref="IPendingUpdatesEvaluator"/>: one pass over the fleet that
|
||||
/// answers both halves of "is anything waiting to be deployed?" — per-site artifact
|
||||
/// staleness and per-instance configuration staleness.
|
||||
///
|
||||
/// <para>
|
||||
/// <b>How it stays affordable.</b> Both halves are batch reads, not per-entity loops.
|
||||
/// <see cref="ArtifactDeploymentService.ComputeCurrentArtifactHashesAsync"/> hoists
|
||||
/// the global artifact queries once for the whole fleet, and
|
||||
/// <see cref="DeploymentService.GetInstanceStalenessAsync"/> runs every flatten
|
||||
/// through ONE <see cref="FlattenSession"/> so a template chain shared by N
|
||||
/// instances is walked once. It is still a whole-fleet flatten, which is why the
|
||||
/// caller caches it behind a TTL rather than calling it per request.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Exceptions propagate.</b> Nothing is swallowed and no partial snapshot is
|
||||
/// invented: the caller's single-flight cache discards a faulted flight and the UI
|
||||
/// renders <i>unavailable</i>. Degrading a failed read to "up to date" is the one
|
||||
/// answer that must never be manufactured — it would silently hide a fleet waiting
|
||||
/// on a deploy.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class PendingUpdatesEvaluator : IPendingUpdatesEvaluator
|
||||
{
|
||||
private readonly ISiteRepository _siteRepository;
|
||||
private readonly ITemplateEngineRepository _templateRepository;
|
||||
private readonly ArtifactDeploymentService _artifactDeploymentService;
|
||||
private readonly DeploymentService _deploymentService;
|
||||
private readonly ILogger<PendingUpdatesEvaluator> _logger;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new evaluator.
|
||||
/// </summary>
|
||||
/// <param name="siteRepository">Repository the site list is read from.</param>
|
||||
/// <param name="templateRepository">Repository the instance list is read from.</param>
|
||||
/// <param name="artifactDeploymentService">Computes each site's current artifact revision hash.</param>
|
||||
/// <param name="deploymentService">Evaluates instance staleness in one batched flatten session.</param>
|
||||
/// <param name="logger">Logger instance.</param>
|
||||
public PendingUpdatesEvaluator(
|
||||
ISiteRepository siteRepository,
|
||||
ITemplateEngineRepository templateRepository,
|
||||
ArtifactDeploymentService artifactDeploymentService,
|
||||
DeploymentService deploymentService,
|
||||
ILogger<PendingUpdatesEvaluator> logger)
|
||||
{
|
||||
_siteRepository = siteRepository;
|
||||
_templateRepository = templateRepository;
|
||||
_artifactDeploymentService = artifactDeploymentService;
|
||||
_deploymentService = deploymentService;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<PendingUpdatesSnapshot> EvaluateAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
|
||||
var sites = await _siteRepository.GetAllSitesAsync(cancellationToken);
|
||||
var instances = await _templateRepository.GetAllInstancesAsync(cancellationToken);
|
||||
|
||||
var currentArtifactHashes =
|
||||
await _artifactDeploymentService.ComputeCurrentArtifactHashesAsync(sites, cancellationToken);
|
||||
|
||||
// Only instances the operator has actually deployed can be stale; a
|
||||
// NotDeployed instance has nothing to compare against, and flattening it
|
||||
// would be pure cost on every TTL expiry.
|
||||
var deployedInstances = instances.Where(i => i.State != InstanceState.NotDeployed).ToList();
|
||||
var staleness = await _deploymentService.GetInstanceStalenessAsync(
|
||||
deployedInstances.Select(i => i.Id).ToList(), cancellationToken);
|
||||
|
||||
var rows = new List<SitePendingUpdates>(sites.Count);
|
||||
foreach (var site in sites.OrderBy(s => s.Name, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
var (deployed, stale, unknown) = TallyInstances(site.Id, deployedInstances, staleness);
|
||||
rows.Add(new SitePendingUpdates(
|
||||
site.Id,
|
||||
site.SiteIdentifier,
|
||||
site.Name,
|
||||
ResolveArtifactState(site, currentArtifactHashes),
|
||||
site.ArtifactsDeployedAt,
|
||||
deployed,
|
||||
stale,
|
||||
unknown));
|
||||
}
|
||||
|
||||
var snapshot = new PendingUpdatesSnapshot(DateTimeOffset.UtcNow, rows);
|
||||
|
||||
// One line per evaluation — this runs at most once per cache TTL, so it is
|
||||
// the cheapest record of what the fleet looked like and what the sweep cost.
|
||||
_logger.LogInformation(
|
||||
"Pending-updates snapshot: {SiteCount} site(s), {DeployedInstanceCount} deployed instance(s), " +
|
||||
"{StaleInstanceCount} stale instance(s), {StaleArtifactSiteCount} site(s) with out-of-date " +
|
||||
"artifacts, in {ElapsedMs} ms",
|
||||
rows.Count,
|
||||
rows.Sum(r => r.DeployedInstanceCount),
|
||||
snapshot.StaleInstanceCount,
|
||||
snapshot.StaleArtifactSiteCount,
|
||||
stopwatch.ElapsedMilliseconds);
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Decides a site's artifact state by comparing the hash stamped by its last
|
||||
/// successful artifact deploy against the hash the current definitions would
|
||||
/// produce. A site with no stamp has never had artifacts deployed, which is
|
||||
/// <see cref="ArtifactDeploymentState.NeverDeployed"/>, NOT out of date. A site
|
||||
/// missing from the computed map (it appeared between the two reads) is reported
|
||||
/// out of date, the side that prompts an operator to look rather than hiding it.
|
||||
/// </summary>
|
||||
/// <param name="site">The site to classify.</param>
|
||||
/// <param name="currentArtifactHashes">Site id to the hash a deploy would send right now.</param>
|
||||
/// <returns>The site's artifact deployment state.</returns>
|
||||
private static ArtifactDeploymentState ResolveArtifactState(
|
||||
Site site,
|
||||
IReadOnlyDictionary<int, string> currentArtifactHashes)
|
||||
{
|
||||
if (site.ArtifactRevisionHash is null)
|
||||
return ArtifactDeploymentState.NeverDeployed;
|
||||
|
||||
return currentArtifactHashes.TryGetValue(site.Id, out var currentHash)
|
||||
&& string.Equals(site.ArtifactRevisionHash, currentHash, StringComparison.Ordinal)
|
||||
? ArtifactDeploymentState.Current
|
||||
: ArtifactDeploymentState.OutOfDate;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Counts one site's deployed, stale and un-evaluatable instances. Only instances
|
||||
/// present in the staleness result are counted — the batch OMITS an instance with
|
||||
/// no deployed snapshot, which is exactly the "nothing to be stale against"
|
||||
/// exclusion the counts need.
|
||||
/// </summary>
|
||||
/// <param name="siteId">Site whose instances are being tallied.</param>
|
||||
/// <param name="deployedInstances">All instances in a state other than NotDeployed.</param>
|
||||
/// <param name="staleness">Instance id to stale (true) / current (false) / unknown (null).</param>
|
||||
/// <returns>The deployed, stale and unknown counts for the site.</returns>
|
||||
private static (int Deployed, int Stale, int Unknown) TallyInstances(
|
||||
int siteId,
|
||||
IReadOnlyList<Instance> deployedInstances,
|
||||
IReadOnlyDictionary<int, bool?> staleness)
|
||||
{
|
||||
var deployed = 0;
|
||||
var stale = 0;
|
||||
var unknown = 0;
|
||||
|
||||
foreach (var instance in deployedInstances)
|
||||
{
|
||||
if (instance.SiteId != siteId)
|
||||
continue;
|
||||
if (!staleness.TryGetValue(instance.Id, out var isStale))
|
||||
continue;
|
||||
|
||||
deployed++;
|
||||
if (isStale == true)
|
||||
stale++;
|
||||
else if (isStale is null)
|
||||
unknown++;
|
||||
}
|
||||
|
||||
return (deployed, stale, unknown);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.DeploymentManager.PendingUpdates;
|
||||
|
||||
/// <summary>
|
||||
/// State of a site's system-wide artifact set (shared scripts, external systems,
|
||||
/// database connections, data connections) relative to the definitions central holds
|
||||
/// right now.
|
||||
/// </summary>
|
||||
public enum ArtifactDeploymentState
|
||||
{
|
||||
/// <summary>
|
||||
/// Artifacts have never been deployed to this site (the site row carries no
|
||||
/// revision hash). Deliberately NOT a pending update: nothing has drifted, the
|
||||
/// operator simply has not deployed yet. Surfaced on the Sites page so a
|
||||
/// freshly-created site is obvious, never counted in the fleet totals or
|
||||
/// escalated by the rail indicator.
|
||||
/// </summary>
|
||||
NeverDeployed,
|
||||
|
||||
/// <summary>
|
||||
/// The hash stamped by the last successful artifact deploy equals the hash the
|
||||
/// current definitions would produce — nothing to deploy.
|
||||
/// </summary>
|
||||
Current,
|
||||
|
||||
/// <summary>
|
||||
/// The current definitions hash differently from what was last deployed, so the
|
||||
/// site is running stale artifacts and a redeploy is pending.
|
||||
/// </summary>
|
||||
OutOfDate
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One site's row in a <see cref="PendingUpdatesSnapshot"/>: its artifact state plus
|
||||
/// the instance staleness tallies that back the Sites-page badge and the rail
|
||||
/// indicator's breakdown.
|
||||
/// </summary>
|
||||
/// <param name="SiteId">Database identifier of the site.</param>
|
||||
/// <param name="SiteIdentifier">Machine-readable site identifier (as used in routing).</param>
|
||||
/// <param name="SiteName">Human-readable site display name.</param>
|
||||
/// <param name="ArtifactState">Whether this site's system-wide artifacts are current.</param>
|
||||
/// <param name="ArtifactsDeployedAt">
|
||||
/// UTC instant of the last successful artifact deploy to this site, or
|
||||
/// <see langword="null"/> when artifacts have never been deployed. Rendered as the
|
||||
/// badge tooltip.
|
||||
/// </param>
|
||||
/// <param name="DeployedInstanceCount">
|
||||
/// Instances at this site that HAVE a deployed configuration snapshot — the
|
||||
/// denominator of "N of M instances out of date". An instance the operator has never
|
||||
/// deployed has nothing to be stale against and is not counted here.
|
||||
/// </param>
|
||||
/// <param name="StaleInstanceCount">
|
||||
/// Deployed instances at this site whose current flatten differs from what is
|
||||
/// deployed, i.e. a redeploy is pending.
|
||||
/// </param>
|
||||
/// <param name="UnknownInstanceCount">
|
||||
/// Deployed instances at this site whose staleness could NOT be evaluated (their
|
||||
/// flatten failed — a missing template, a broken binding). Never counted as stale:
|
||||
/// the fleet totals must not invent a pending update out of a read failure.
|
||||
/// </param>
|
||||
public sealed record SitePendingUpdates(
|
||||
int SiteId,
|
||||
string SiteIdentifier,
|
||||
string SiteName,
|
||||
ArtifactDeploymentState ArtifactState,
|
||||
DateTimeOffset? ArtifactsDeployedAt,
|
||||
int DeployedInstanceCount,
|
||||
int StaleInstanceCount,
|
||||
int UnknownInstanceCount);
|
||||
|
||||
/// <summary>
|
||||
/// One fleet-wide answer to "is anything waiting to be deployed?" — every site's
|
||||
/// artifact state and instance staleness tallies as of a single instant.
|
||||
///
|
||||
/// <para>
|
||||
/// Computed by <see cref="IPendingUpdatesEvaluator"/> and held behind the Central
|
||||
/// UI's single-flight TTL cache, so however many circuits are watching, the fleet is
|
||||
/// flattened at most once per TTL. An immutable value: a scoped view is taken with
|
||||
/// <see cref="ForSites"/> rather than by mutating the snapshot.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
/// <param name="ComputedAt">
|
||||
/// UTC instant the snapshot was computed. Rendered as the "as of HH:mm UTC" caption
|
||||
/// and preserved verbatim through <see cref="ForSites"/> — a scoped view is the same
|
||||
/// observation, not a newer one.
|
||||
/// </param>
|
||||
/// <param name="Sites">Per-site rows, ordered by site name (ordinal, ignore-case).</param>
|
||||
public sealed record PendingUpdatesSnapshot(
|
||||
DateTimeOffset ComputedAt,
|
||||
IReadOnlyList<SitePendingUpdates> Sites)
|
||||
{
|
||||
/// <summary>
|
||||
/// Total instances across the snapshot's sites whose deployed configuration has
|
||||
/// drifted from their current flatten. Instances whose staleness could not be
|
||||
/// evaluated are excluded.
|
||||
/// </summary>
|
||||
public int StaleInstanceCount => Sites.Sum(s => s.StaleInstanceCount);
|
||||
|
||||
/// <summary>
|
||||
/// Number of sites running out-of-date system-wide artifacts.
|
||||
/// <see cref="ArtifactDeploymentState.NeverDeployed"/> is deliberately NOT counted
|
||||
/// — the operator has not deployed yet, which is not a pending update.
|
||||
/// </summary>
|
||||
public int StaleArtifactSiteCount =>
|
||||
Sites.Count(s => s.ArtifactState == ArtifactDeploymentState.OutOfDate);
|
||||
|
||||
/// <summary>
|
||||
/// Whether anything in this snapshot is waiting to be deployed — the single
|
||||
/// predicate the rail indicator switches on.
|
||||
/// </summary>
|
||||
public bool HasPendingUpdates => StaleInstanceCount > 0 || StaleArtifactSiteCount > 0;
|
||||
|
||||
/// <summary>
|
||||
/// Returns a copy of this snapshot keeping only the named sites, preserving
|
||||
/// <see cref="ComputedAt"/>. Used to narrow the fleet snapshot to a site-scoped
|
||||
/// operator's permitted sites, so the computed totals never include a site the
|
||||
/// user cannot see.
|
||||
/// </summary>
|
||||
/// <param name="siteIds">Database identifiers of the sites to keep.</param>
|
||||
/// <returns>A snapshot over the retained sites, in the same order.</returns>
|
||||
public PendingUpdatesSnapshot ForSites(IReadOnlySet<int> siteIds)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(siteIds);
|
||||
return new PendingUpdatesSnapshot(
|
||||
ComputedAt,
|
||||
Sites.Where(s => siteIds.Contains(s.SiteId)).ToList());
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Transport;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Templates;
|
||||
using ZB.MOM.WW.ScadaBridge.DeploymentManager.PendingUpdates;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.DeploymentManager;
|
||||
|
||||
@@ -69,6 +70,13 @@ public static class ServiceCollectionExtensions
|
||||
// flattened config has drifted from their deployed snapshot. Scoped so it
|
||||
// shares the per-request DbContext with the in-flight import.
|
||||
services.AddScoped<IStaleInstanceProbe, StaleInstanceProbe>();
|
||||
|
||||
// Fleet-wide "is anything waiting to be deployed?" aggregate behind the
|
||||
// Central UI rail indicator and the Sites-page artifact badges. Scoped because
|
||||
// it reads through the same per-request DbContext as the services it composes;
|
||||
// the Central UI's singleton single-flight cache runs it in its own DI scope so
|
||||
// the fleet is flattened at most once per cache TTL.
|
||||
services.AddScoped<IPendingUpdatesEvaluator, PendingUpdatesEvaluator>();
|
||||
return services;
|
||||
}
|
||||
|
||||
|
||||
+22
@@ -72,4 +72,26 @@ public class DeploymentManagerOptionsValidatorTests
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("SiteDeploymentReconcileTimeout", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ZeroPendingUpdatesCacheTtl_IsRejected()
|
||||
{
|
||||
// A zero TTL makes every Deployer/Administrator circuit's poll a cache miss,
|
||||
// so the whole fleet is re-flattened on each one — the exact cost the cache
|
||||
// exists to avoid.
|
||||
var result = Validate(new DeploymentManagerOptions { PendingUpdatesCacheTtl = TimeSpan.Zero });
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("PendingUpdatesCacheTtl", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NegativePendingUpdatesCacheTtl_IsRejected()
|
||||
{
|
||||
var result = Validate(new DeploymentManagerOptions
|
||||
{
|
||||
PendingUpdatesCacheTtl = TimeSpan.FromSeconds(-1)
|
||||
});
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("PendingUpdatesCacheTtl", result.FailureMessage);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,415 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Transport;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Templates;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication;
|
||||
using ZB.MOM.WW.ScadaBridge.DeploymentManager.PendingUpdates;
|
||||
using ZB.MOM.WW.ScadaBridge.TemplateEngine.Flattening;
|
||||
using ZB.MOM.WW.ScadaBridge.TemplateEngine.Validation;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.DeploymentManager.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Coverage for <see cref="PendingUpdatesEvaluator"/> — the single fleet-wide read
|
||||
/// behind the Central UI rail indicator and the Sites page badges.
|
||||
///
|
||||
/// <para>
|
||||
/// The evaluator is built over the REAL <see cref="ArtifactDeploymentService"/> and
|
||||
/// <see cref="DeploymentService"/> (substituting only their repository and flattening
|
||||
/// seams, as the sibling suites do), because the two properties worth pinning are
|
||||
/// exactly the ones a mocked collaborator would paper over: that the artifact state
|
||||
/// is decided by comparing the site's STORED hash against a hash recomputed from the
|
||||
/// current definitions, and that only instances the operator has actually deployed
|
||||
/// are ever handed to the staleness batch.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class PendingUpdatesEvaluatorTests
|
||||
{
|
||||
private readonly ISiteRepository _siteRepo = Substitute.For<ISiteRepository>();
|
||||
private readonly ITemplateEngineRepository _templateRepo = Substitute.For<ITemplateEngineRepository>();
|
||||
private readonly IExternalSystemRepository _externalSystemRepo = Substitute.For<IExternalSystemRepository>();
|
||||
private readonly INotificationRepository _notificationRepo = Substitute.For<INotificationRepository>();
|
||||
private readonly IDeploymentManagerRepository _deploymentRepo = Substitute.For<IDeploymentManagerRepository>();
|
||||
private readonly IFlatteningPipeline _pipeline = Substitute.For<IFlatteningPipeline>();
|
||||
private readonly IAuditService _audit = Substitute.For<IAuditService>();
|
||||
private readonly RevisionHashService _hasher = new();
|
||||
private readonly ArtifactRevisionHashService _artifactHasher = new();
|
||||
|
||||
private readonly ArtifactDeploymentService _artifactService;
|
||||
private readonly DeploymentService _deploymentService;
|
||||
private readonly PendingUpdatesEvaluator _evaluator;
|
||||
|
||||
/// <summary>
|
||||
/// Builds the evaluator over the real services; every repository and the
|
||||
/// flattening pipeline are substituted.
|
||||
/// </summary>
|
||||
public PendingUpdatesEvaluatorTests()
|
||||
{
|
||||
_pipeline.CreateSession().Returns(new FlattenSession(new TemplateGraphWatermark()));
|
||||
|
||||
var comms = new CommunicationService(
|
||||
Options.Create(new CommunicationOptions()),
|
||||
NullLogger<CommunicationService>.Instance);
|
||||
|
||||
_artifactService = new ArtifactDeploymentService(
|
||||
_siteRepo, _deploymentRepo, _templateRepo, _externalSystemRepo, _notificationRepo,
|
||||
comms, _audit,
|
||||
Options.Create(new DeploymentManagerOptions()),
|
||||
NullLogger<ArtifactDeploymentService>.Instance,
|
||||
_artifactHasher);
|
||||
|
||||
_deploymentService = new DeploymentService(
|
||||
_deploymentRepo,
|
||||
_siteRepo,
|
||||
_pipeline,
|
||||
comms,
|
||||
new OperationLockManager(),
|
||||
_audit,
|
||||
new DiffService(),
|
||||
_hasher,
|
||||
new DeploymentStatusNotifier(NullLogger<DeploymentStatusNotifier>.Instance),
|
||||
Substitute.For<IStaleInstanceProbe>(),
|
||||
Options.Create(new DeploymentManagerOptions()),
|
||||
Options.Create(new CommunicationOptions()),
|
||||
NullLogger<DeploymentService>.Instance);
|
||||
|
||||
_evaluator = new PendingUpdatesEvaluator(
|
||||
_siteRepo, _templateRepo, _artifactService, _deploymentService,
|
||||
NullLogger<PendingUpdatesEvaluator>.Instance);
|
||||
}
|
||||
|
||||
// ── arrange helpers ──────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>Stubs the site list read.</summary>
|
||||
private void ArrangeSites(params Site[] sites) =>
|
||||
_siteRepo.GetAllSitesAsync(Arg.Any<CancellationToken>()).Returns(sites);
|
||||
|
||||
/// <summary>Stubs the instance list read.</summary>
|
||||
private void ArrangeInstances(params Instance[] instances) =>
|
||||
_templateRepo.GetAllInstancesAsync(Arg.Any<CancellationToken>()).Returns(instances);
|
||||
|
||||
/// <summary>Builds an instance in the given lifecycle state at the given site.</summary>
|
||||
private static Instance MakeInstance(int id, int siteId, InstanceState state) =>
|
||||
new($"Inst{id}") { Id = id, SiteId = siteId, State = state };
|
||||
|
||||
/// <summary>Builds a one-attribute flattened config with the given scalar value.</summary>
|
||||
private static FlattenedConfiguration ScalarConfig(string value) => new()
|
||||
{
|
||||
InstanceUniqueName = "TestInst",
|
||||
Attributes = [new ResolvedAttribute { CanonicalName = "Setpoint", Value = value, DataType = "Int32" }]
|
||||
};
|
||||
|
||||
/// <summary>Stubs a successful flatten of <paramref name="instanceId"/> at the given hash.</summary>
|
||||
private void ArrangeFlatten(int instanceId, FlattenedConfiguration config, string hash) =>
|
||||
_pipeline.FlattenAndValidateAsync(
|
||||
instanceId, Arg.Any<CancellationToken>(), Arg.Any<bool>(), Arg.Any<FlattenSession?>())
|
||||
.Returns(Result<FlatteningPipelineResult>.Success(
|
||||
new FlatteningPipelineResult(config, hash, ValidationResult.Success())));
|
||||
|
||||
/// <summary>Stubs a FAILED flatten of <paramref name="instanceId"/> — the "unknown" leg.</summary>
|
||||
private void ArrangeFlattenFailure(int instanceId) =>
|
||||
_pipeline.FlattenAndValidateAsync(
|
||||
instanceId, Arg.Any<CancellationToken>(), Arg.Any<bool>(), Arg.Any<FlattenSession?>())
|
||||
.Returns(Result<FlatteningPipelineResult>.Failure("template missing"));
|
||||
|
||||
/// <summary>Stubs the (InstanceId, RevisionHash) projection read.</summary>
|
||||
private void ArrangeDeployedHashes(params (int InstanceId, string Hash)[] rows) =>
|
||||
_deploymentRepo.GetDeployedRevisionHashesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(rows.ToDictionary(r => r.InstanceId, r => r.Hash));
|
||||
|
||||
/// <summary>Stubs the fat snapshot read for <paramref name="instanceId"/>.</summary>
|
||||
private void ArrangeSnapshot(int instanceId, FlattenedConfiguration config, string frozenHash) =>
|
||||
_deploymentRepo.GetDeployedSnapshotByInstanceIdAsync(instanceId, Arg.Any<CancellationToken>())
|
||||
.Returns(new DeployedConfigSnapshot("dep1", frozenHash, JsonSerializer.Serialize(config))
|
||||
{
|
||||
InstanceId = instanceId,
|
||||
DeployedAt = DateTimeOffset.UtcNow
|
||||
});
|
||||
|
||||
/// <summary>
|
||||
/// Asks the artifact service what the site's hash WOULD be right now — the value a
|
||||
/// successful deploy would have stamped, i.e. what makes a site read Current.
|
||||
/// </summary>
|
||||
private async Task<string> CurrentArtifactHashAsync(Site site) =>
|
||||
(await _artifactService.ComputeCurrentArtifactHashesAsync([site]))[site.Id];
|
||||
|
||||
// ── artifact state ───────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task EvaluateAsync_NullStoredHash_IsNeverDeployedAndNotAPendingUpdate()
|
||||
{
|
||||
var site = new Site("Alpha", "site-a") { Id = 1 };
|
||||
ArrangeSites(site);
|
||||
ArrangeInstances();
|
||||
|
||||
var snapshot = await _evaluator.EvaluateAsync();
|
||||
|
||||
var row = Assert.Single(snapshot.Sites);
|
||||
Assert.Equal(ArtifactDeploymentState.NeverDeployed, row.ArtifactState);
|
||||
Assert.Null(row.ArtifactsDeployedAt);
|
||||
// NeverDeployed is the operator not having deployed yet, not a pending
|
||||
// update: it is surfaced on the Sites page but never counted or nagged about.
|
||||
Assert.Equal(0, snapshot.StaleArtifactSiteCount);
|
||||
Assert.False(snapshot.HasPendingUpdates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EvaluateAsync_StoredHashMatchesCurrentDefinitions_IsCurrent()
|
||||
{
|
||||
var site = new Site("Alpha", "site-a") { Id = 1 };
|
||||
var deployedAt = DateTimeOffset.UtcNow.AddHours(-2);
|
||||
site.ArtifactRevisionHash = await CurrentArtifactHashAsync(site);
|
||||
site.ArtifactsDeployedAt = deployedAt;
|
||||
ArrangeSites(site);
|
||||
ArrangeInstances();
|
||||
|
||||
var snapshot = await _evaluator.EvaluateAsync();
|
||||
|
||||
var row = Assert.Single(snapshot.Sites);
|
||||
Assert.Equal(ArtifactDeploymentState.Current, row.ArtifactState);
|
||||
Assert.Equal(deployedAt, row.ArtifactsDeployedAt);
|
||||
Assert.Equal(0, snapshot.StaleArtifactSiteCount);
|
||||
Assert.False(snapshot.HasPendingUpdates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EvaluateAsync_StoredHashDiffersFromCurrentDefinitions_IsOutOfDateAndCounted()
|
||||
{
|
||||
var site = new Site("Alpha", "site-a") { Id = 1 };
|
||||
site.ArtifactRevisionHash = "a-hash-from-an-older-artifact-set";
|
||||
site.ArtifactsDeployedAt = DateTimeOffset.UtcNow.AddDays(-3);
|
||||
ArrangeSites(site);
|
||||
ArrangeInstances();
|
||||
|
||||
var snapshot = await _evaluator.EvaluateAsync();
|
||||
|
||||
var row = Assert.Single(snapshot.Sites);
|
||||
Assert.Equal(ArtifactDeploymentState.OutOfDate, row.ArtifactState);
|
||||
Assert.Equal(1, snapshot.StaleArtifactSiteCount);
|
||||
Assert.True(snapshot.HasPendingUpdates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EvaluateAsync_CarriesSiteIdentityOntoTheRow()
|
||||
{
|
||||
var site = new Site("Alpha Plant", "site-a") { Id = 7 };
|
||||
ArrangeSites(site);
|
||||
ArrangeInstances();
|
||||
|
||||
var snapshot = await _evaluator.EvaluateAsync();
|
||||
|
||||
var row = Assert.Single(snapshot.Sites);
|
||||
Assert.Equal(7, row.SiteId);
|
||||
Assert.Equal("site-a", row.SiteIdentifier);
|
||||
Assert.Equal("Alpha Plant", row.SiteName);
|
||||
}
|
||||
|
||||
// ── instance counts ──────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task EvaluateAsync_CountsDeployedStaleAndUnknownInstancesPerSite()
|
||||
{
|
||||
var site = new Site("Alpha", "site-a") { Id = 1 };
|
||||
ArrangeSites(site);
|
||||
// 1 = deployed + stale, 2 = deployed but un-evaluatable, 3 = never deployed.
|
||||
ArrangeInstances(
|
||||
MakeInstance(1, 1, InstanceState.Enabled),
|
||||
MakeInstance(2, 1, InstanceState.Disabled),
|
||||
MakeInstance(3, 1, InstanceState.NotDeployed));
|
||||
|
||||
var deployedConfig = ScalarConfig("5");
|
||||
var deployedHash = _hasher.ComputeHash(deployedConfig);
|
||||
ArrangeDeployedHashes((1, deployedHash), (2, deployedHash));
|
||||
ArrangeFlatten(1, ScalarConfig("9"), _hasher.ComputeHash(ScalarConfig("9")));
|
||||
ArrangeSnapshot(1, deployedConfig, deployedHash);
|
||||
ArrangeFlattenFailure(2);
|
||||
|
||||
var snapshot = await _evaluator.EvaluateAsync();
|
||||
|
||||
var row = Assert.Single(snapshot.Sites);
|
||||
Assert.Equal(2, row.DeployedInstanceCount);
|
||||
Assert.Equal(1, row.StaleInstanceCount);
|
||||
Assert.Equal(1, row.UnknownInstanceCount);
|
||||
Assert.Equal(1, snapshot.StaleInstanceCount);
|
||||
Assert.True(snapshot.HasPendingUpdates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EvaluateAsync_NotDeployedInstancesAreNeverHandedToTheStalenessBatch()
|
||||
{
|
||||
var site = new Site("Alpha", "site-a") { Id = 1 };
|
||||
ArrangeSites(site);
|
||||
ArrangeInstances(
|
||||
MakeInstance(1, 1, InstanceState.Enabled),
|
||||
MakeInstance(2, 1, InstanceState.NotDeployed));
|
||||
|
||||
var config = ScalarConfig("5");
|
||||
var hash = _hasher.ComputeHash(config);
|
||||
// Instance 2 deliberately HAS a leftover deployed-hash row (an undeploy that
|
||||
// left the snapshot behind), so the staleness batch would happily evaluate it
|
||||
// if its id were passed — the filter under test is the State gate, not the
|
||||
// batch's own omission of instances with no snapshot.
|
||||
ArrangeDeployedHashes((1, hash), (2, hash));
|
||||
ArrangeFlatten(1, config, hash);
|
||||
ArrangeFlatten(2, config, hash);
|
||||
|
||||
var snapshot = await _evaluator.EvaluateAsync();
|
||||
|
||||
// A NotDeployed instance has nothing to be stale against; flattening it would
|
||||
// be pure cost on every TTL expiry, so it must never reach the batch.
|
||||
await _pipeline.Received(1).FlattenAndValidateAsync(
|
||||
1, Arg.Any<CancellationToken>(), Arg.Any<bool>(), Arg.Any<FlattenSession?>());
|
||||
await _pipeline.DidNotReceive().FlattenAndValidateAsync(
|
||||
2, Arg.Any<CancellationToken>(), Arg.Any<bool>(), Arg.Any<FlattenSession?>());
|
||||
Assert.Equal(1, snapshot.Sites.Single().DeployedInstanceCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EvaluateAsync_AttributesInstanceCountsToTheirOwnSite()
|
||||
{
|
||||
var alpha = new Site("Alpha", "site-a") { Id = 1 };
|
||||
var bravo = new Site("Bravo", "site-b") { Id = 2 };
|
||||
ArrangeSites(alpha, bravo);
|
||||
ArrangeInstances(
|
||||
MakeInstance(1, 1, InstanceState.Enabled),
|
||||
MakeInstance(2, 2, InstanceState.Enabled));
|
||||
|
||||
var deployedConfig = ScalarConfig("5");
|
||||
var deployedHash = _hasher.ComputeHash(deployedConfig);
|
||||
ArrangeDeployedHashes((1, deployedHash), (2, deployedHash));
|
||||
ArrangeFlatten(1, deployedConfig, deployedHash);
|
||||
ArrangeFlatten(2, ScalarConfig("9"), _hasher.ComputeHash(ScalarConfig("9")));
|
||||
ArrangeSnapshot(2, deployedConfig, deployedHash);
|
||||
|
||||
var snapshot = await _evaluator.EvaluateAsync();
|
||||
|
||||
Assert.Equal(0, snapshot.Sites.Single(s => s.SiteId == 1).StaleInstanceCount);
|
||||
Assert.Equal(1, snapshot.Sites.Single(s => s.SiteId == 2).StaleInstanceCount);
|
||||
Assert.Equal(1, snapshot.StaleInstanceCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EvaluateAsync_NothingStale_HasNoPendingUpdates()
|
||||
{
|
||||
var site = new Site("Alpha", "site-a") { Id = 1 };
|
||||
site.ArtifactRevisionHash = await CurrentArtifactHashAsync(site);
|
||||
ArrangeSites(site);
|
||||
ArrangeInstances(MakeInstance(1, 1, InstanceState.Enabled));
|
||||
|
||||
var config = ScalarConfig("5");
|
||||
var hash = _hasher.ComputeHash(config);
|
||||
ArrangeDeployedHashes((1, hash));
|
||||
ArrangeFlatten(1, config, hash);
|
||||
|
||||
var snapshot = await _evaluator.EvaluateAsync();
|
||||
|
||||
Assert.Equal(1, snapshot.Sites.Single().DeployedInstanceCount);
|
||||
Assert.Equal(0, snapshot.StaleInstanceCount);
|
||||
Assert.Equal(0, snapshot.StaleArtifactSiteCount);
|
||||
Assert.False(snapshot.HasPendingUpdates);
|
||||
}
|
||||
|
||||
// ── ordering, ComputedAt, ForSites ───────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task EvaluateAsync_OrdersSitesByNameIgnoringCase()
|
||||
{
|
||||
ArrangeSites(
|
||||
new Site("charlie", "site-c") { Id = 3 },
|
||||
new Site("Alpha", "site-a") { Id = 1 },
|
||||
new Site("bravo", "site-b") { Id = 2 });
|
||||
ArrangeInstances();
|
||||
|
||||
var snapshot = await _evaluator.EvaluateAsync();
|
||||
|
||||
Assert.Equal(["Alpha", "bravo", "charlie"], snapshot.Sites.Select(s => s.SiteName));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EvaluateAsync_StampsComputedAt()
|
||||
{
|
||||
var before = DateTimeOffset.UtcNow;
|
||||
ArrangeSites(new Site("Alpha", "site-a") { Id = 1 });
|
||||
ArrangeInstances();
|
||||
|
||||
var snapshot = await _evaluator.EvaluateAsync();
|
||||
|
||||
Assert.InRange(snapshot.ComputedAt, before, DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EvaluateAsync_ExceptionsPropagate()
|
||||
{
|
||||
_siteRepo.GetAllSitesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns<IReadOnlyList<Site>>(_ => throw new InvalidOperationException("db down"));
|
||||
|
||||
// The caller's single-flight cache discards a faulted flight and the UI
|
||||
// renders "unavailable"; swallowing here would make a broken fleet read
|
||||
// "up to date", which is the one answer that must never be invented.
|
||||
var ex = await Assert.ThrowsAsync<InvalidOperationException>(() => _evaluator.EvaluateAsync());
|
||||
Assert.Equal("db down", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ForSites_KeepsOnlyTheNamedSitesAndTheSameComputedAt()
|
||||
{
|
||||
var computedAt = DateTimeOffset.UtcNow.AddMinutes(-1);
|
||||
var snapshot = new PendingUpdatesSnapshot(computedAt,
|
||||
[
|
||||
new SitePendingUpdates(1, "site-a", "Alpha", ArtifactDeploymentState.OutOfDate, null, 2, 2, 0),
|
||||
new SitePendingUpdates(2, "site-b", "Bravo", ArtifactDeploymentState.Current, null, 1, 1, 0),
|
||||
new SitePendingUpdates(3, "site-c", "Charlie", ArtifactDeploymentState.OutOfDate, null, 0, 0, 0)
|
||||
]);
|
||||
|
||||
var scoped = snapshot.ForSites(new HashSet<int> { 2 });
|
||||
|
||||
Assert.Equal(computedAt, scoped.ComputedAt);
|
||||
var row = Assert.Single(scoped.Sites);
|
||||
Assert.Equal(2, row.SiteId);
|
||||
// The totals are recomputed over the kept sites, so a site-scoped user is
|
||||
// never shown a count that includes a site they cannot see.
|
||||
Assert.Equal(1, scoped.StaleInstanceCount);
|
||||
Assert.Equal(0, scoped.StaleArtifactSiteCount);
|
||||
Assert.True(scoped.HasPendingUpdates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ForSites_EmptyScope_YieldsNoPendingUpdates()
|
||||
{
|
||||
var snapshot = new PendingUpdatesSnapshot(DateTimeOffset.UtcNow,
|
||||
[
|
||||
new SitePendingUpdates(1, "site-a", "Alpha", ArtifactDeploymentState.OutOfDate, null, 2, 2, 0)
|
||||
]);
|
||||
|
||||
var scoped = snapshot.ForSites(new HashSet<int>());
|
||||
|
||||
Assert.Empty(scoped.Sites);
|
||||
Assert.False(scoped.HasPendingUpdates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Snapshot_Totals_SumAcrossSites()
|
||||
{
|
||||
var snapshot = new PendingUpdatesSnapshot(DateTimeOffset.UtcNow,
|
||||
[
|
||||
new SitePendingUpdates(1, "site-a", "Alpha", ArtifactDeploymentState.OutOfDate, null, 4, 2, 1),
|
||||
new SitePendingUpdates(2, "site-b", "Bravo", ArtifactDeploymentState.NeverDeployed, null, 0, 0, 0),
|
||||
new SitePendingUpdates(3, "site-c", "Charlie", ArtifactDeploymentState.Current, null, 3, 3, 0)
|
||||
]);
|
||||
|
||||
Assert.Equal(5, snapshot.StaleInstanceCount);
|
||||
// NeverDeployed is deliberately not counted alongside OutOfDate.
|
||||
Assert.Equal(1, snapshot.StaleArtifactSiteCount);
|
||||
Assert.True(snapshot.HasPendingUpdates);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user